pi-codex-search 0.1.2 → 0.1.4

package/src/config.ts

@@ -3,7 +3,9 @@ import { homedir } from "node:os";
 import { dirname, join } from "node:path";
 import type { SearchContextSize } from "./codex.ts";
 
-export type Freshness = "live" | "cached";
+export type SearchApi = "standalone" | "responses";
+
+export type Freshness = "live" | "cached" | "indexed";
 
 export type ConfigScope = "project" | "home";
 
@@ -15,6 +17,9 @@ export interface PiCodexSearchConfig {
   clientVersion?: string;
   searchContextSize?: SearchContextSize;
   freshness?: Freshness;
+  searchApi?: SearchApi;
+  standaloneEnabled?: boolean;
+  batchSize?: number;
 }
 
 export interface ResolvedConfig {
@@ -25,6 +30,9 @@ export interface ResolvedConfig {
   clientVersion?: string;
   defaultSearchContextSize: SearchContextSize;
   defaultFreshness: Freshness;
+  searchApi: SearchApi;
+  standaloneEnabled: boolean;
+  batchSize: number;
   sources: {
     project?: PiCodexSearchConfig;
     home?: PiCodexSearchConfig;
@@ -36,19 +44,40 @@ export const DEFAULT_ENABLED = true;
 export const DEFAULT_TOOL_NAME = "codex_search";
 export const DEFAULT_SEARCH_CONTEXT_SIZE: SearchContextSize = "medium";
 export const DEFAULT_FRESHNESS: Freshness = "live";
+export const DEFAULT_SEARCH_API: SearchApi = "responses";
+export const DEFAULT_STANDALONE_ENABLED = false;
+export const STANDALONE_TOOL_NAME = "codex_standalone_web";
+export const DEFAULT_BATCH_SIZE = 5;
 export const CONFIG_FILE_NAME = "pi-codex-search.json";
 const TOOL_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]{0,63}$/;
 const CONTEXT_SIZES: readonly SearchContextSize[] = ["low", "medium", "high"] as const;
-const FRESHNESS_VALUES: readonly Freshness[] = ["live", "cached"] as const;
+const FRESHNESS_VALUES: readonly Freshness[] = ["live", "cached", "indexed"] as const;
+const SEARCH_API_VALUES: readonly SearchApi[] = ["standalone", "responses"] as const;
+export const MIN_BATCH_SIZE = 1;
+export const MAX_BATCH_SIZE = 32;
 
 export function getConfigPath(scope: ConfigScope, cwd: string): string {
   if (scope === "project") return join(cwd, ".pi", CONFIG_FILE_NAME);
   return join(homedir(), ".pi", CONFIG_FILE_NAME);
 }
 
-export async function loadConfig(cwd: string): Promise<ResolvedConfig> {
+export function isProjectTrustedContext(ctx: unknown): boolean {
+  if (ctx === null || ctx === undefined || typeof ctx !== "object") return true;
+  const maybe = ctx as { isProjectTrusted?: unknown };
+  if (typeof maybe.isProjectTrusted === "boolean") return maybe.isProjectTrusted;
+  if (typeof maybe.isProjectTrusted !== "function") return true;
+  try {
+    return Boolean(maybe.isProjectTrusted());
+  } catch {
+    return false;
+  }
+}
+
+export async function loadConfig(cwd: string, isProjectTrusted = true): Promise<ResolvedConfig> {
   const homeConfig = await readConfigFile(getConfigPath("home", cwd));
-  const projectConfig = await readConfigFile(getConfigPath("project", cwd));
+  const projectConfig = isProjectTrusted
+    ? await readConfigFile(getConfigPath("project", cwd))
+    : undefined;
   const envConfig = readEnvConfig();
 
   const merged: PiCodexSearchConfig = {
@@ -62,6 +91,11 @@ export async function loadConfig(cwd: string): Promise<ResolvedConfig> {
     toolName: merged.toolName ?? DEFAULT_TOOL_NAME,
     defaultSearchContextSize: merged.searchContextSize ?? DEFAULT_SEARCH_CONTEXT_SIZE,
     defaultFreshness: merged.freshness ?? DEFAULT_FRESHNESS,
+    searchApi: DEFAULT_SEARCH_API,
+    standaloneEnabled:
+      merged.standaloneEnabled ??
+      (merged.searchApi === "standalone" ? true : DEFAULT_STANDALONE_ENABLED),
+    batchSize: merged.batchSize ?? DEFAULT_BATCH_SIZE,
     sources: {},
   };
   if (merged.model !== undefined) resolved.model = merged.model;
@@ -138,6 +172,12 @@ function readEnvConfig(): PiCodexSearchConfig | undefined {
   }
   const freshness = trimmedEnv("PI_CODEX_WEB_SEARCH_FRESHNESS");
   if (freshness !== undefined) env.freshness = freshness as Freshness;
+  const searchApi = trimmedEnv("PI_CODEX_WEB_SEARCH_API");
+  if (searchApi !== undefined) env.searchApi = searchApi as SearchApi;
+  const standaloneEnabled = booleanEnv("PI_CODEX_WEB_STANDALONE_ENABLED");
+  if (standaloneEnabled !== undefined) env.standaloneEnabled = standaloneEnabled;
+  const batchSize = integerEnv("PI_CODEX_WEB_SEARCH_BATCH_SIZE");
+  if (batchSize !== undefined) env.batchSize = batchSize;
 
   if (Object.keys(env).length === 0) return undefined;
   validateConfig(env, "<env>");
@@ -177,6 +217,29 @@ function validateConfig(config: PiCodexSearchConfig, sourceLabel: string): void
         `Expected one of ${FRESHNESS_VALUES.join(", ")}.`,
     );
   }
+  if (config.standaloneEnabled !== undefined && typeof config.standaloneEnabled !== "boolean") {
+    throw new Error(
+      `Invalid standaloneEnabled in ${sourceLabel}: ${JSON.stringify(config.standaloneEnabled)}. Must be a boolean.`,
+    );
+  }
+  if (config.searchApi !== undefined && !SEARCH_API_VALUES.includes(config.searchApi)) {
+    throw new Error(
+      `Invalid searchApi in ${sourceLabel}: ${JSON.stringify(config.searchApi)}. ` +
+        `Expected one of ${SEARCH_API_VALUES.join(", ")}.`,
+    );
+  }
+  if (config.batchSize !== undefined) {
+    if (
+      !Number.isInteger(config.batchSize) ||
+      config.batchSize < MIN_BATCH_SIZE ||
+      config.batchSize > MAX_BATCH_SIZE
+    ) {
+      throw new Error(
+        `Invalid batchSize in ${sourceLabel}: ${JSON.stringify(config.batchSize)}. ` +
+          `Expected an integer between ${MIN_BATCH_SIZE} and ${MAX_BATCH_SIZE}.`,
+      );
+    }
+  }
 }
 
 function trimmedEnv(name: string): string | undefined {
@@ -195,6 +258,16 @@ function booleanEnv(name: string): boolean | undefined {
   throw new Error(`Invalid ${name}: ${JSON.stringify(raw)}. Expected 'true' or 'false'.`);
 }
 
+function integerEnv(name: string): number | undefined {
+  const raw = trimmedEnv(name);
+  if (raw === undefined) return undefined;
+  const parsed = Number(raw);
+  if (!Number.isInteger(parsed)) {
+    throw new Error(`Invalid ${name}: ${JSON.stringify(raw)}. Expected an integer.`);
+  }
+  return parsed;
+}
+
 function isPlainObject(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }

package/src/cookies.ts

@@ -0,0 +1,131 @@
+/**
+ * Process-local Cloudflare cookie store for ChatGPT endpoints.
+ *
+ * Mirrors the constraints in codex chatgpt_cloudflare_cookies.rs:
+ * - HTTPS only
+ * - ChatGPT host allowlist
+ * - Cloudflare infrastructure cookie name allowlist only
+ * - Never store account/session/auth cookies
+ */
+
+export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+interface Cookie {
+  name: string;
+  value: string;
+  domain: string;
+  path: string;
+  secure: boolean;
+}
+
+const ALLOWED_HOSTS = ["chatgpt.com", "chat.openai.com", "chatgpt-staging.com"];
+const HOST_SUFFIXES = [".chatgpt.com", ".chatgpt-staging.com"];
+
+const ALLOWED_COOKIE_NAMES = new Set([
+  "__cf_bm",
+  "__cflb",
+  "__cfruid",
+  "__cfseq",
+  "__cfwaitingroom",
+  "_cfuvid",
+  "cf_clearance",
+  "cf_ob_info",
+  "cf_use_ob",
+]);
+
+function isAllowedHost(host: string): boolean {
+  if (ALLOWED_HOSTS.includes(host)) return true;
+  return HOST_SUFFIXES.some((suffix) => host.endsWith(suffix));
+}
+
+function isAllowedCookieName(name: string): boolean {
+  if (ALLOWED_COOKIE_NAMES.has(name)) return true;
+  return name.startsWith("cf_chl_");
+}
+
+function parseSetCookieHeader(header: string, url: URL): Cookie | undefined {
+  const [firstPart] = header.split(";");
+  if (!firstPart) return undefined;
+  const eqIndex = firstPart.indexOf("=");
+  if (eqIndex <= 0) return undefined;
+  const name = firstPart.slice(0, eqIndex).trim();
+  const value = firstPart.slice(eqIndex + 1).trim();
+  if (!isAllowedCookieName(name)) return undefined;
+  return {
+    name,
+    value,
+    domain: url.hostname,
+    path: "/",
+    secure: true,
+  };
+}
+
+function cookieKey(cookie: Cookie): string {
+  return `${cookie.domain}:${cookie.path}:${cookie.name}`;
+}
+
+export class ChatGptCloudflareCookieStore {
+  private cookies = new Map<string, Cookie>();
+
+  setCookies(setCookieHeaders: string[], url: URL): void {
+    if (url.protocol !== "https:") return;
+    if (!isAllowedHost(url.hostname)) return;
+    for (const header of setCookieHeaders) {
+      const cookie = parseSetCookieHeader(header, url);
+      if (cookie) {
+        this.cookies.set(cookieKey(cookie), cookie);
+      }
+    }
+  }
+
+  cookiesForUrl(url: URL): string | undefined {
+    if (url.protocol !== "https:") return undefined;
+    if (!isAllowedHost(url.hostname)) return undefined;
+    const parts: string[] = [];
+    for (const cookie of this.cookies.values()) {
+      if (cookie.domain === url.hostname) {
+        parts.push(`${cookie.name}=${cookie.value}`);
+      }
+    }
+    return parts.length > 0 ? parts.join("; ") : undefined;
+  }
+}
+
+const SHARED_STORE = new ChatGptCloudflareCookieStore();
+
+export function getSharedCookieStore(): ChatGptCloudflareCookieStore {
+  return SHARED_STORE;
+}
+
+export function wrapFetchWithCookies(fetchImpl: FetchLike): FetchLike {
+  return async (input, init) => {
+    const url = new URL(
+      typeof input === "string" ? input : input instanceof URL ? input.href : input.url,
+    );
+    const cookieHeader = SHARED_STORE.cookiesForUrl(url);
+    const headers = new Headers(input instanceof Request ? input.headers : undefined);
+    new Headers(init?.headers).forEach((value, key) => headers.set(key, value));
+    if (cookieHeader) {
+      const existing = headers.get("cookie");
+      headers.set("cookie", existing ? `${existing}; ${cookieHeader}` : cookieHeader);
+    }
+
+    const response = await fetchImpl(input, { ...init, headers });
+
+    const setCookie =
+      response.headers.getSetCookie?.() ?? parseSetCookieLegacy(response.headers.get("set-cookie"));
+    if (setCookie.length > 0) {
+      SHARED_STORE.setCookies(setCookie, url);
+    }
+    return response;
+  };
+}
+
+function parseSetCookieLegacy(value: string | null): string[] {
+  if (!value) return [];
+  // Split on comma, but Set-Cookie values rarely contain bare commas in CF cookies.
+  return value
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
+}

package/src/errors.ts

@@ -0,0 +1,56 @@
+export type CodexErrorKind = "auth" | "rate_limit" | "transport" | "timeout" | "schema" | "unknown";
+
+export class CodexError extends Error {
+  readonly kind: CodexErrorKind;
+  readonly status?: number;
+
+  constructor(kind: CodexErrorKind, message: string, status?: number) {
+    super(message);
+    this.name = "CodexError";
+    this.kind = kind;
+    if (status !== undefined) this.status = status;
+  }
+}
+
+export function classifyError(error: unknown): CodexErrorKind {
+  if (error instanceof CodexError) return error.kind;
+  if (error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError")) {
+    return "timeout";
+  }
+  return "unknown";
+}
+
+export function classifyHttpStatus(status: number): CodexErrorKind {
+  if (status === 401 || status === 403) return "auth";
+  if (status === 429) return "rate_limit";
+  return "transport";
+}
+
+export function formatHttpErrorBody(text: string, mode: "responses" | "standalone"): string {
+  if (isCloudflareChallenge(text)) {
+    const advice =
+      mode === "standalone"
+        ? "Use codex_search for search, or retry after Codex/ChatGPT has refreshed its Cloudflare clearance."
+        : "Retry after Codex/ChatGPT has refreshed its Cloudflare clearance.";
+    return `Cloudflare challenge blocked the Codex request. ${advice}`;
+  }
+  return text;
+}
+
+export function isCloudflareChallenge(text: string): boolean {
+  const lower = text.toLowerCase();
+  return (
+    lower.includes("/cdn-cgi/challenge-platform/") ||
+    lower.includes("cf_chl_") ||
+    lower.includes("enable javascript and cookies to continue")
+  );
+}
+
+export function classifyEventErrorMessage(message: string): CodexErrorKind {
+  const lower = message.toLowerCase();
+  if (/rate[- ]?limit|too many requests|quota|429/.test(lower)) return "rate_limit";
+  if (/auth|unauthori[sz]ed|forbidden|401|403/.test(lower)) return "auth";
+  if (/timeout|timed out/.test(lower)) return "timeout";
+  if (/network|connection|disconnect|transport|fetch failed/.test(lower)) return "transport";
+  return "unknown";
+}

package/src/modes/responses.ts

@@ -0,0 +1,310 @@
+import {
+  CodexError,
+  classifyEventErrorMessage,
+  classifyHttpStatus,
+  formatHttpErrorBody,
+} from "../errors.ts";
+import type { CodexTransport } from "../transport.ts";
+import type {
+  CodexWebSearchResult,
+  CodexCitation,
+  CodexSearchCall,
+  SearchContextSize,
+} from "./types.ts";
+
+export interface ResponsesSearchOptions {
+  query: string;
+  model: string;
+  transport: CodexTransport;
+  externalWebAccess: boolean;
+  indexGatedWebAccess?: true;
+  searchContextSize?: SearchContextSize;
+  sessionId?: string;
+  threadId?: string;
+  signal?: AbortSignal;
+  onTextDelta?: (delta: string) => void;
+}
+
+interface SseEvent {
+  type: string;
+  data?: unknown;
+  raw?: string;
+}
+
+interface ResponseOutputText {
+  type?: string;
+  text?: string;
+  annotations?: Array<{
+    type?: string;
+    title?: string;
+    url?: string;
+    start_index?: number;
+    end_index?: number;
+  }>;
+}
+
+interface ResponseOutputItem {
+  id?: string;
+  type?: string;
+  status?: string;
+  role?: string;
+  action?: {
+    type?: string;
+    query?: string;
+    queries?: string[];
+    url?: string;
+  };
+  content?: ResponseOutputText[];
+}
+
+interface ResponseUsage {
+  input_tokens?: number;
+  output_tokens?: number;
+  total_tokens?: number;
+}
+
+interface ResponseEnvelope {
+  id?: string;
+  usage?: ResponseUsage;
+}
+
+interface ResponseEventData {
+  response?: ResponseEnvelope;
+  item?: ResponseOutputItem;
+  delta?: string;
+  error?: {
+    message?: string;
+    code?: string;
+  };
+}
+
+export async function runResponsesSearch(
+  options: ResponsesSearchOptions,
+): Promise<CodexWebSearchResult> {
+  const {
+    transport,
+    query,
+    model,
+    externalWebAccess,
+    searchContextSize,
+    sessionId,
+    threadId,
+    signal,
+    onTextDelta,
+  } = options;
+  const headers = transport.buildHeaders("text/event-stream");
+  if (sessionId) headers.set("session-id", sessionId);
+  if (threadId) {
+    headers.set("thread-id", threadId);
+    headers.set("x-client-request-id", threadId);
+  }
+
+  const webSearchTool: Record<string, unknown> = {
+    type: "web_search",
+    external_web_access: externalWebAccess,
+    search_context_size: searchContextSize ?? "medium",
+  };
+
+  const response = await transport.fetch(transport.resolveEndpoint("responses"), {
+    method: "POST",
+    headers,
+    body: JSON.stringify({
+      model,
+      instructions:
+        "You are a concise web search assistant. Use web search, answer the query, and preserve source citations from annotations.",
+      input: [
+        {
+          type: "message",
+          role: "user",
+          content: [{ type: "input_text", text: query }],
+        },
+      ],
+      tools: [webSearchTool],
+      tool_choice: "required",
+      parallel_tool_calls: true,
+      store: false,
+      stream: true,
+      include: [],
+    }),
+    signal,
+  });
+
+  if (!response.ok) {
+    const status = response.status;
+    const text = formatHttpErrorBody(await response.text(), "responses");
+    throw new CodexError(
+      classifyHttpStatus(status),
+      `Codex responses request failed: HTTP ${status}: ${text}`,
+      status,
+    );
+  }
+  if (!response.body) {
+    throw new Error("Codex responses response did not include a body");
+  }
+
+  let responseId: string | undefined;
+  let usage: ResponseUsage | undefined;
+  let streamedText = "";
+  const messageTextParts: string[] = [];
+  const searchCalls = new Map<string, CodexSearchCall>();
+  const citations = new Map<string, CodexCitation>();
+
+  for await (const event of parseSse(response.body)) {
+    const data = event.data as ResponseEventData | undefined;
+    if (!data) continue;
+
+    if (event.type === "response.created") {
+      responseId = data.response?.id;
+      continue;
+    }
+
+    if (event.type === "response.output_text.delta") {
+      const delta = data.delta ?? "";
+      streamedText += delta;
+      onTextDelta?.(delta);
+      continue;
+    }
+
+    if (event.type === "response.output_item.added" && data.item?.type === "web_search_call") {
+      const item = data.item;
+      if (item.id) {
+        searchCalls.set(item.id, {
+          id: item.id,
+          status: item.status,
+        });
+      }
+      continue;
+    }
+
+    if (event.type === "response.output_item.done") {
+      collectOutputItem(data.item, searchCalls, messageTextParts, citations);
+      continue;
+    }
+
+    if (event.type === "response.completed") {
+      usage = data.response?.usage;
+      continue;
+    }
+
+    if (event.type === "response.failed") {
+      const message = data.error?.message ?? data.error?.code ?? "Codex web search failed";
+      throw new CodexError(classifyEventErrorMessage(message), message);
+    }
+  }
+
+  return {
+    responseId,
+    model,
+    text: messageTextParts.join("") || streamedText,
+    searchCalls: [...searchCalls.values()],
+    citations: [...citations.values()],
+    usage: usage
+      ? {
+          inputTokens: usage.input_tokens,
+          outputTokens: usage.output_tokens,
+          totalTokens: usage.total_tokens,
+        }
+      : undefined,
+  };
+}
+
+async function* parseSse(body: ReadableStream<Uint8Array>): AsyncGenerator<SseEvent> {
+  const reader = body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+
+  let doneReading = false;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        doneReading = true;
+        break;
+      }
+      buffer += decoder.decode(value, { stream: true });
+
+      let separator = findSseSeparator(buffer);
+      while (separator) {
+        const frame = buffer.slice(0, separator.index);
+        buffer = buffer.slice(separator.index + separator.length);
+        const event = parseSseFrame(frame);
+        if (event) yield event;
+        separator = findSseSeparator(buffer);
+      }
+    }
+  } finally {
+    if (!doneReading) await reader.cancel().catch(() => undefined);
+    reader.releaseLock();
+  }
+
+  buffer += decoder.decode();
+  const event = parseSseFrame(buffer);
+  if (event) yield event;
+}
+
+function findSseSeparator(buffer: string): { index: number; length: number } | undefined {
+  const match = /\r?\n\r?\n/.exec(buffer);
+  return match?.index === undefined ? undefined : { index: match.index, length: match[0].length };
+}
+
+function parseSseFrame(frame: string): SseEvent | undefined {
+  const lines = frame.split(/\r?\n/);
+  let type = "";
+  const dataLines: string[] = [];
+
+  for (const line of lines) {
+    if (line.startsWith("event:")) {
+      type = line.slice("event:".length).trim();
+    } else if (line.startsWith("data:")) {
+      dataLines.push(line.slice("data:".length).trimStart());
+    }
+  }
+
+  if (dataLines.length === 0) return undefined;
+  const raw = dataLines.join("\n");
+  if (raw === "[DONE]") return undefined;
+
+  try {
+    return { type, data: JSON.parse(raw) };
+  } catch {
+    return { type, raw };
+  }
+}
+
+function collectOutputItem(
+  item: ResponseOutputItem | undefined,
+  searchCalls: Map<string, CodexSearchCall>,
+  messageTextParts: string[],
+  citations: Map<string, CodexCitation>,
+): void {
+  if (!item) return;
+
+  if (item.type === "web_search_call") {
+    const key = item.id ?? `search-${searchCalls.size + 1}`;
+    const query = item.action?.query ?? item.action?.queries?.join(", ");
+    searchCalls.set(key, {
+      id: item.id,
+      status: item.status,
+      query,
+      url: item.action?.url,
+      actionType: item.action?.type,
+    });
+    return;
+  }
+
+  if (item.type !== "message" || item.role !== "assistant") return;
+
+  for (const part of item.content ?? []) {
+    if (part.type !== "output_text") continue;
+    messageTextParts.push(part.text ?? "");
+    for (const annotation of part.annotations ?? []) {
+      if (annotation.type !== "url_citation" || !annotation.url) continue;
+      citations.set(annotation.url, {
+        title: annotation.title,
+        url: annotation.url,
+        startIndex: annotation.start_index,
+        endIndex: annotation.end_index,
+      });
+    }
+  }
+}