pi-twitterapi.io 0.1.1

package/src/twitterapi/http.ts

@@ -0,0 +1,340 @@
+import { isObject, type FetchLike } from "./core.js";
+
+/**
+ * Upper bound on any single retry delay. A server-supplied `Retry-After` longer
+ * than this is not clamped-and-retried-early — that would spend the remaining
+ * attempts while the limit is still in force — the call fails visibly instead.
+ */
+export const MAX_RETRY_DELAY_MS = 60_000;
+
+/** Default ceiling for the adaptive page budget (see `maxPagesCeiling`). */
+export const DEFAULT_MAX_PAGES_CEILING = 20;
+
+/** Parse `Retry-After` in either HTTP form (delta-seconds or HTTP-date). */
+export function parseRetryAfter(value: string | null, nowMs: number = Date.now()): number | undefined {
+  if (!value) return undefined;
+  const trimmed = value.trim();
+  if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1_000;
+  const when = Date.parse(trimmed);
+  if (Number.isNaN(when)) return undefined;
+  return Math.max(0, when - nowMs);
+}
+
+function retryDelayMs(attempt: number, baseDelayMs: number, retryAfterMs: number | undefined): number {
+  if (retryAfterMs !== undefined) {
+    // Never shorten a server-supplied delay: retrying earlier than asked just
+    // spends the remaining attempts while the limit is still in force. If the
+    // request is beyond what we are willing to wait, stop and say so.
+    if (retryAfterMs > MAX_RETRY_DELAY_MS) {
+      throw new Error(
+        `twitterapi.io asked to retry in ${Math.ceil(retryAfterMs / 1_000)}s, beyond the ` +
+          `${MAX_RETRY_DELAY_MS / 1_000}s cap; not retrying. Try again later or lower the request rate.`,
+      );
+    }
+    // Floor at the backoff base so `Retry-After: 0` cannot become a hot loop.
+    return Math.max(retryAfterMs, baseDelayMs);
+  }
+  const backoff = baseDelayMs * 2 ** attempt;
+  return Math.min(backoff + Math.random() * baseDelayMs, MAX_RETRY_DELAY_MS);
+}
+
+interface AttemptResult {
+  response: Response;
+  body: unknown;
+  /** Attempts spent to obtain this result (1 unless retries were used). */
+  attempts: number;
+  /** Set when the response arrived but its body could not be read. */
+  bodyError?: string;
+}
+
+/**
+ * Marks a transport-level failure (fetch rejection, stalled/socket error while
+ * reading the body). Distinguished from a caller cancellation, a timeout, and
+ * a JSON syntax error so the retry decision does not depend on message text.
+ */
+export class TransportError extends Error {
+  constructor(message: string, options?: { cause?: unknown }) {
+    super(message, options);
+    this.name = "TransportError";
+  }
+}
+
+/** Marks a per-request timeout. Distinct from cancellation, and retryable. */
+export class TimeoutError extends Error {
+  constructor(message: string, options?: { cause?: unknown }) {
+    super(message, options);
+    this.name = "TimeoutError";
+  }
+}
+
+/** Marks a caller cancellation. Never retried. */
+export class CancelledError extends Error {
+  constructor(message = "twitterapi.io search was cancelled") {
+    super(message);
+    this.name = "CancelledError";
+  }
+}
+
+/** One request attempt with its own timeout; returns the parsed body alongside the response. */
+async function requestOnce(
+  url: string,
+  apiKey: string,
+  fetcher: FetchLike,
+  timeoutMs: number,
+  outerSignal: AbortSignal | undefined,
+): Promise<AttemptResult> {
+  const controller = new AbortController();
+  const onAbort = () => controller.abort(outerSignal?.reason);
+  if (outerSignal?.aborted) onAbort();
+  else outerSignal?.addEventListener("abort", onAbort, { once: true });
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  // The deadline is enforced here rather than delegated to the fetcher: a
+  // fetcher or body stream that ignores the signal must still not hang the call.
+  const aborted = new Promise<never>((_resolve, reject) => {
+    const rejectAbort = () => reject(new Error("twitterapi.io request aborted"));
+    if (controller.signal.aborted) rejectAbort();
+    else controller.signal.addEventListener("abort", rejectAbort, { once: true });
+  });
+  aborted.catch(() => {}); // settled-after-race rejections are not unhandled errors
+  try {
+    const response = await Promise.race([
+      fetcher(url, { headers: { "X-API-Key": apiKey }, signal: controller.signal }),
+      aborted,
+    ]);
+    let body: unknown;
+    let bodyError: string | undefined;
+    try {
+      body = await Promise.race([response.json(), aborted]);
+    } catch (error) {
+      // A stalled body trips the same abort as a stalled fetch, so distinguish
+      // transport/cancellation failures from a genuine JSON syntax error.
+      if (outerSignal?.aborted) throw new CancelledError();
+      // The response ARRIVED; only its body stalled. This must NOT surface as a
+      // retryable timeout: the page may already have been billed, so the status
+      // decides what happens next rather than the retry classifier.
+      if (controller.signal.aborted) {
+        bodyError = `the response body did not finish within ${timeoutMs}ms`;
+      } else if (!(error instanceof SyntaxError)) {
+        // A JSON syntax failure means "no usable body"; any other read failure is
+        // recorded for the same reason.
+        bodyError = error instanceof Error ? error.message : String(error);
+      }
+      body = undefined;
+    }
+    if (controller.signal.aborted && !bodyError) {
+      if (outerSignal?.aborted) throw new CancelledError();
+      throw new TimeoutError(`twitterapi.io request timed out after ${timeoutMs}ms`);
+    }
+    return { response, body, attempts: 1, bodyError };
+  } catch (error) {
+    if (outerSignal?.aborted) throw new CancelledError();
+    if (controller.signal.aborted) throw new TimeoutError(`twitterapi.io request timed out after ${timeoutMs}ms`);
+    if (error instanceof TransportError || error instanceof TimeoutError) throw error;
+    throw new TransportError(error instanceof Error ? error.message : String(error), { cause: error });
+  } finally {
+    clearTimeout(timeout);
+    outerSignal?.removeEventListener("abort", onAbort);
+  }
+}
+
+/** Only transport failures and timeouts are retried; cancellations never are. */
+function isRetryableError(error: Error): boolean {
+  return error instanceof TransportError || error instanceof TimeoutError;
+}
+
+/** Backoff timer that clears itself when the caller cancels. */
+export function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    const cleanup = () => {
+      clearTimeout(timer);
+      signal?.removeEventListener("abort", onAbort);
+    };
+    const onAbort = () => {
+      cleanup();
+      reject(new CancelledError());
+    };
+    const timer = setTimeout(() => {
+      cleanup();
+      resolve();
+    }, ms);
+    // Installed before any await so an already-aborted signal settles at once.
+    if (signal?.aborted) onAbort();
+    else signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+/** Abortable backoff: a cancelled tool call must not sit out the remaining delay. */
+export async function sleepAbortable(
+  ms: number,
+  signal: AbortSignal | undefined,
+  sleep: (ms: number, signal?: AbortSignal) => Promise<void>,
+): Promise<void> {
+  if (signal?.aborted) throw new CancelledError();
+  await sleep(ms, signal);
+}
+
+/**
+ * The documented safe retry scope: 429 and 503 only. Retrying other statuses
+ * repeats a request the upstream may already have billed, and the provider's
+ * own guidance names these two. Widening this is a deliberate policy change.
+ */
+export function isRetryableStatus(status: number): boolean {
+  return status === 429 || status === 503;
+}
+
+/**
+ * Detail text from the several error envelopes the upstream returns in the
+ * wild: `{detail}` and `{msg}` for endpoint errors, and
+ * `{error, message}` for account-level ones — a 402 out of credits arrives as
+ * `{"error":"Unauthorized","message":"Credits is not enough.Please recharge"}`.
+ * Without the last two keys, a billing failure reports only "HTTP 402".
+ */
+export function errorDetail(body: Record<string, unknown>): string | undefined {
+  for (const key of ["detail", "msg", "message", "error"]) {
+    const value = body[key];
+    if (typeof value === "string" && value.trim()) return value.trim();
+  }
+  return undefined;
+}
+
+/**
+ * Validate a payload that may have failed, in the order that preserves the most
+ * useful error: HTTP status first (an unsuccessful response keeps its status and
+ * retry context even when the body carries a semantic envelope), then an
+ * unreadable body, then shape, then a semantic `status: "error"` envelope.
+ */
+export function ensureSuccessfulPayload(
+  response: Response,
+  body: unknown,
+  bodyError: string | undefined,
+  attempts: number,
+): Record<string, unknown> {
+  if (!response.ok) {
+    throw new Error(describeHttpFailure(response.status, isObject(body) ? errorDetail(body) : undefined, attempts));
+  }
+  // A successful status whose body could not be read is NOT retried: the
+  // upstream may already have billed the page, so retrying risks paying twice.
+  if (bodyError) {
+    throw new Error(
+      `twitterapi.io returned HTTP ${response.status} but its body could not be read (${bodyError}); ` +
+        "not retrying because the request may already have been billed.",
+    );
+  }
+  if (!isObject(body)) {
+    throw new Error(`twitterapi.io returned a malformed response (HTTP ${response.status})`);
+  }
+  // A semantic failure can also arrive with HTTP 200.
+  if (body.status === "error") {
+    throw new Error(`twitterapi.io error: ${errorDetail(body) ?? "unknown"}`);
+  }
+  return body;
+}
+
+/** Options shared by every twitterapi.io endpoint: retry, pacing, cancellation. */
+export interface TwitterApiRequestOptions {
+  /** Per-request timeout in ms (default 30_000). */
+  timeoutMs?: number;
+  /** Retry attempts for 429/503 responses (default 3). */
+  maxRetries?: number;
+  /** Base delay for exponential backoff in ms (default 5_000, the unpaid-tier floor). */
+  retryBaseDelayMs?: number;
+  /** Minimum spacing between successive upstream requests in ms (default 5_000). */
+  minRequestIntervalMs?: number;
+  /** Injected clock, for tests. */
+  now?: () => number;
+  /** Injected sleep, for tests. Receives the cancellation signal so a custom sleep can honor it. */
+  sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+  /** Caller cancellation (Pi passes the tool's AbortSignal). */
+  signal?: AbortSignal;
+}
+
+/** Resolved retry/pacing settings, so each endpoint builds them identically. */
+interface RequestSettings {
+  timeoutMs: number;
+  maxRetries: number;
+  retryBaseDelayMs: number;
+  minRequestIntervalMs: number;
+  now: () => number;
+  sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
+  signal?: AbortSignal;
+}
+
+export function resolveRequestSettings(options: TwitterApiRequestOptions): RequestSettings {
+  const timeoutMs = options.timeoutMs ?? 30_000;
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new Error("timeoutMs must be a positive number");
+  return {
+    timeoutMs,
+    maxRetries: Math.max(0, options.maxRetries ?? 3),
+    retryBaseDelayMs: Math.max(0, options.retryBaseDelayMs ?? 5_000),
+    minRequestIntervalMs: Math.max(0, options.minRequestIntervalMs ?? 5_000),
+    now: options.now ?? Date.now,
+    sleep: options.sleep ?? defaultSleep,
+    signal: options.signal,
+  };
+}
+
+/** Human-readable HTTP failure that keeps the status, detail, and retry context. */
+export function describeHttpFailure(status: number, detail: string | undefined, attempts: number): string {
+  const lead = status === 429
+    ? "rate limited"
+    : status === 503
+      ? "temporarily unavailable"
+      : status === 402
+        ? "payment required"
+        : "error";
+  const suffix = attempts > 1 ? ` after ${attempts} attempts` : "";
+  return `twitterapi.io ${lead}${detail ? `: ${detail}` : ""} (HTTP ${status}${suffix})`;
+}
+
+/** Fetch with retry/backoff on 429/503 (the free tier rate-limits bursts). */
+export async function requestWithRetry(
+  url: string,
+  apiKey: string,
+  fetcher: FetchLike,
+  options: {
+    maxRetries: number;
+    retryBaseDelayMs: number;
+    timeoutMs: number;
+    sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
+    signal?: AbortSignal;
+  },
+): Promise<AttemptResult> {
+  let lastError: Error | undefined;
+  for (let attempt = 0; attempt <= options.maxRetries; attempt += 1) {
+    let result: AttemptResult;
+    try {
+      result = await requestOnce(url, apiKey, fetcher, options.timeoutMs, options.signal);
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      if (options.signal?.aborted || attempt === options.maxRetries || !isRetryableError(lastError)) {
+        throw lastError;
+      }
+      await sleepAbortable(retryDelayMs(attempt, options.retryBaseDelayMs, undefined), options.signal, options.sleep);
+      continue;
+    }
+
+    if (!isRetryableStatus(result.response.status) || attempt === options.maxRetries) {
+      return { ...result, attempts: attempt + 1 };
+    }
+    lastError = new Error(`twitterapi.io error: HTTP ${result.response.status}`);
+    const retryAfterMs = parseRetryAfter(result.response.headers?.get?.("retry-after") ?? null);
+    await sleepAbortable(retryDelayMs(attempt, options.retryBaseDelayMs, retryAfterMs), options.signal, options.sleep);
+  }
+  throw lastError ?? new Error("twitterapi.io request failed");
+}
+
+/** Page bound shared with the configuration range, so a valid config cannot fail here. */
+export const MAX_PAGE_BOUND = 100;
+
+/**
+ * Page/count bounds. An absent value takes the default; a supplied one that is
+ * out of range is an error rather than a silent fallback, so a caller asking for
+ * 0 pages is told instead of quietly getting the default.
+ */
+export function boundedCount(value: unknown, fallback: number, max: number, name: string): number {
+  if (value === undefined) return fallback;
+  if (typeof value !== "number" || !Number.isInteger(value) || value < 1 || value > max) {
+    throw new Error(`twitter ${name} must be an integer between 1 and ${max} (got ${String(value)})`);
+  }
+  return value;
+}

package/src/twitterapi/params.ts

@@ -0,0 +1,81 @@
+import type { NormalizedSearchParams, TwitterApiSearchParams } from "./core.js";
+
+const HANDLE_RE = /^@?([A-Za-z0-9_]{1,15})$/;
+
+function normalizeHandles(handles: string[] | undefined, field: string): string[] {
+  if (!handles) return [];
+  if (handles.length > 20) throw new Error(`twitter ${field} accepts at most 20 handles (got ${handles.length})`);
+  return handles.map((handle) => {
+    const match = HANDLE_RE.exec(handle.trim());
+    if (!match) throw new Error(`twitter ${field} must be valid X handles without spaces (got "${handle}")`);
+    return match[1];
+  });
+}
+
+function normalizeDate(date: string | undefined, field: string): string | undefined {
+  if (!date) return undefined;
+  const trimmed = date.trim();
+  const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(trimmed);
+  if (!match) throw new Error(`twitter ${field} must be YYYY-MM-DD (got "${date}")`);
+  const year = Number(match[1]);
+  const month = Number(match[2]);
+  const day = Number(match[3]);
+  const daysInMonth = [31, isLeapYear(year) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
+  if (month < 1 || month > 12 || day < 1 || day > daysInMonth[month - 1]) {
+    throw new Error(`twitter ${field} is not a real calendar date (got "${date}")`);
+  }
+  return trimmed;
+}
+
+function isLeapYear(year: number): boolean {
+  return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
+}
+
+export function normalizeParams(params: TwitterApiSearchParams): NormalizedSearchParams {
+  const query = params.query?.trim();
+  if (!query) throw new Error("twitter query must not be empty");
+
+  const allowed = normalizeHandles(params.allowed_x_handles, "allowed_x_handles");
+  const excluded = normalizeHandles(params.excluded_x_handles, "excluded_x_handles");
+  if (allowed.length > 0 && excluded.length > 0) {
+    throw new Error("twitter allowed_x_handles and excluded_x_handles cannot be set together");
+  }
+
+  const fromDate = normalizeDate(params.from_date, "from_date");
+  const toDate = normalizeDate(params.to_date, "to_date");
+  if (fromDate && toDate && fromDate > toDate) {
+    throw new Error("twitter from_date must be before or equal to to_date");
+  }
+
+  const count = params.count ?? 10;
+  if (!Number.isInteger(count) || count < 1 || count > 50) {
+    throw new Error("twitter count must be an integer between 1 and 50");
+  }
+
+  const queryType = params.queryType ?? "Latest";
+  if (queryType !== "Latest" && queryType !== "Top") {
+    throw new Error('twitter queryType must be "Latest" or "Top"');
+  }
+
+  return { query, allowed_x_handles: allowed, excluded_x_handles: excluded, from_date: fromDate, to_date: toDate, queryType, count };
+}
+
+/** Build the X advanced-search expression sent to twitterapi.io. */
+export function buildExpression(params: NormalizedSearchParams): string {
+  const constraints: string[] = [];
+  if (params.allowed_x_handles?.length) {
+    constraints.push(params.allowed_x_handles.length === 1
+      ? `from:${params.allowed_x_handles[0]}`
+      : `(${params.allowed_x_handles.map((h) => `from:${h}`).join(" OR ")})`);
+  }
+  if (params.excluded_x_handles?.length) {
+    for (const h of params.excluded_x_handles) constraints.push(`-from:${h}`);
+  }
+  if (params.from_date) constraints.push(`since:${params.from_date}`);
+  if (params.to_date) constraints.push(`until:${params.to_date}`);
+  // Group the user query so appended AND-constraints cannot leak into
+  // an OR branch (e.g. `cats OR dogs` + handle must not leave `cats` unscoped).
+  if (constraints.length === 0) return params.query;
+  return `(${params.query}) ${constraints.join(" ")}`;
+}
+

package/src/twitterapi/search.ts

@@ -0,0 +1,233 @@
+import {
+  ADVANCED_SEARCH_PATH,
+  TWITTERAPI_BASE_URL,
+  type FetchLike,
+  type NormalizedSearchParams,
+  type SearchDetails,
+  type SearchTermination,
+  type Tweet,
+} from "./core.js";
+import { buildExpression } from "./params.js";
+import { parseTweetDate, resolveLocalWindow, withPaddedStart } from "./window.js";
+import {
+  CancelledError,
+  DEFAULT_MAX_PAGES_CEILING,
+  defaultSleep,
+  ensureSuccessfulPayload,
+  requestWithRetry,
+  sleepAbortable,
+  type TwitterApiRequestOptions,
+} from "./http.js";
+import { asTweet } from "./tweet.js";
+
+export interface SearchTweetsOptions {
+  /** Max pages to fetch (default 5). Bounds total requests when pages come back empty. */
+  maxPages?: number;
+  /**
+   * Hard ceiling on pages fetched in one search (default 20). `maxPages` is
+   * clamped to it, so an explicit ceiling can never be exceeded.
+   */
+  maxPagesCeiling?: number;
+  /** Per-request timeout in ms (default 30_000). */
+  timeoutMs?: number;
+  /** Retry attempts for 429/503 responses (default 3). */
+  maxRetries?: number;
+  /** Base delay for exponential backoff in ms (default 5_000, the unpaid-tier floor). */
+  retryBaseDelayMs?: number;
+  /**
+   * Minimum spacing between successive upstream requests in ms (default
+   * 5_000). twitterapi.io rate-limits per API key — brand-new unpaid accounts
+   * allow 0.2 QPS, i.e. one request every 5 seconds — and pagination would
+   * otherwise fire requests back to back. Raise it for a higher tier, or set it
+   * to 0 to disable pacing.
+   */
+  minRequestIntervalMs?: number;
+  /** Injected clock, for tests. */
+  now?: () => number;
+  /** Injected sleep, for tests. Receives the cancellation signal so a custom sleep can honor it. */
+  sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+  /**
+   * Caller cancellation (Pi passes the tool's AbortSignal). When set, in-flight
+   * requests and backoff sleeps are abandoned as soon as it aborts.
+   */
+  signal?: AbortSignal;
+  /**
+   * Fixed local UTC offset in minutes. Omit to use the host timezone, which
+   * resolves each boundary with its own DST-correct offset.
+   */
+  localUtcOffsetMinutes?: number;
+}
+/** True when retrieval stopped while the upstream still held more results. */
+export function isTruncated(stoppedBy: SearchTermination): boolean {
+  return stoppedBy === "page-cap" || stoppedBy === "cursor-cycle" || stoppedBy === "cursor-missing";
+}
+
+/**
+ * Decide the next page, or record an honest reason for stopping.
+ *
+ * `has_next_page !== true` genuinely means "no more results". Reporting the same
+ * when upstream says there ARE more pages but hands us no cursor would present a
+ * partial result as complete, so that case counts as truncation instead.
+ */
+export function advanceOrStop(
+  payload: Record<string, unknown>,
+  seenCursors: Set<string>,
+): { cursor: string } | SearchTermination {
+  if (payload.has_next_page !== true) return "exhausted";
+  if (typeof payload.next_cursor !== "string" || !payload.next_cursor) return "cursor-missing";
+  if (seenCursors.has(payload.next_cursor)) return "cursor-cycle";
+  seenCursors.add(payload.next_cursor);
+  return { cursor: payload.next_cursor };
+}
+
+/** Run paginated advanced_search until `count` tweets or no more pages. */
+export async function searchTweets(
+  params: NormalizedSearchParams,
+  apiKey: string,
+  fetcher: FetchLike = fetch,
+  options: SearchTweetsOptions = {},
+): Promise<SearchDetails> {
+  const window = resolveLocalWindow(params, options.localUtcOffsetMinutes);
+  const requestParams = withPaddedStart(params, window);
+  const expression = buildExpression(requestParams);
+  const target = params.count ?? 10;
+  // A ceiling is a hard cap: the base budget is clamped to it, never the other
+  // way round, and non-integer or infinite values fall back to the defaults
+  // rather than removing the termination bound.
+  const requestedCeiling = options.maxPagesCeiling ?? DEFAULT_MAX_PAGES_CEILING;
+  const pageCeiling = Number.isInteger(requestedCeiling) && requestedCeiling >= 1
+    ? requestedCeiling
+    : DEFAULT_MAX_PAGES_CEILING;
+  const requestedBase = options.maxPages ?? 5;
+  const basePageBudget = Math.min(
+    Number.isInteger(requestedBase) && requestedBase >= 1 ? requestedBase : 5,
+    pageCeiling,
+  );
+  // Free pages are granted only for posts NEWER than the window: those form the
+  // upstream 04:00 UTC band sitting in front of the requested day. Older ones
+  // (the start padding) sit at the tail, so paging further gains nothing.
+  const trimBandHours = window?.trimHours ?? 0;
+  let trimmedNewer = 0;
+  let trimmedOlder = 0;
+  // Pages charged to the base budget. A page spent crossing the newer band is
+  // not charged — it was not answering the query — which is what stops a busy
+  // topic from returning nothing. Totals stay bounded by `pageCeiling`.
+  let pagesOnBudget = 0;
+  const maxRetries = Math.max(0, options.maxRetries ?? 3);
+  const retryBaseDelayMs = Math.max(0, options.retryBaseDelayMs ?? 5_000);
+  const minRequestIntervalMs = Math.max(0, options.minRequestIntervalMs ?? 5_000);
+  const now = options.now ?? Date.now;
+  const timeoutMs = options.timeoutMs ?? 30_000;
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new Error("timeoutMs must be a positive number");
+  const sleep = options.sleep ?? defaultSleep;
+  const signal = options.signal;
+  const collected: Tweet[] = [];
+  // Keyed on both identifiers: the same post can come back with an id on one
+  // page and only a permalink on another, which a single `id ?? url` key misses.
+  const seenIds = new Set<string>();
+  const seenUrls = new Set<string>();
+  const seenCursors = new Set<string>();
+  let cursor = "";
+  let pages = 0;
+  let lastRequestAt: number | undefined;
+  // Only an explicit break changes this; falling out of the loop means the page
+  // cap was reached, which is exactly the case that must be disclosed.
+  let stoppedBy: SearchTermination = "page-cap";
+
+  while (collected.length < target && pagesOnBudget < basePageBudget && pages < pageCeiling) {
+    if (signal?.aborted) throw new CancelledError();
+    // Pace successive requests: the per-key QPS ceiling is low enough (0.2 QPS
+    // on an unpaid account) that unpaced pagination can rate-limit itself.
+    if (minRequestIntervalMs > 0 && lastRequestAt !== undefined) {
+      const wait = minRequestIntervalMs - (now() - lastRequestAt);
+      if (wait > 0) await sleepAbortable(wait, signal, sleep);
+    }
+    pages += 1;
+    let url: URL;
+    try {
+      url = new URL(TWITTERAPI_BASE_URL + ADVANCED_SEARCH_PATH);
+    } catch {
+      throw new Error("twitterapi.io: invalid search URL (internal error)");
+    }
+    url.searchParams.set("query", expression);
+    url.searchParams.set("queryType", params.queryType ?? "Latest");
+    if (cursor) url.searchParams.set("cursor", cursor);
+
+    const { response, body, attempts, bodyError } = await requestWithRetry(url.toString(), apiKey, fetcher, {
+      maxRetries,
+      retryBaseDelayMs,
+      timeoutMs,
+      sleep,
+      signal,
+    });
+    lastRequestAt = now();
+
+    // Errors are validated in one place, in the order that preserves the most
+    // useful signal: status, then unreadable body, then shape, then semantics.
+    const payload = ensureSuccessfulPayload(response, body, bodyError, attempts);
+    if (!Array.isArray(payload.tweets)) {
+      throw new Error("twitterapi.io returned a malformed response (missing tweets array)");
+    }
+
+    let pageTrimmedNewer = 0;
+    for (const raw of payload.tweets) {
+      const tweet = asTweet(raw);
+      if (!tweet) continue;
+      // Enforce the caller's real local day range: the upstream window may be
+      // padded at the start and stops short at the end. Unparseable dates are
+      // kept (fail-open).
+      if (window) {
+        const at = parseTweetDate(tweet.createdAt);
+        if (at !== undefined) {
+          if (at >= window.endMs) {
+            pageTrimmedNewer += 1;
+            trimmedNewer += 1;
+            continue;
+          }
+          if (at < window.startMs) {
+            trimmedOlder += 1;
+            continue;
+          }
+        }
+      }
+      // Register both identifiers even when this occurrence is itself a
+      // duplicate: otherwise an alias chain ((id=1,url=A), (id=1,url=B),
+      // (no id,url=B)) lets the third representation through as a new post.
+      const duplicate =
+        (tweet.id !== undefined && seenIds.has(tweet.id)) ||
+        (tweet.url !== undefined && seenUrls.has(tweet.url));
+      if (tweet.id) seenIds.add(tweet.id);
+      if (tweet.url) seenUrls.add(tweet.url);
+      if (duplicate) continue;
+      collected.push(tweet);
+      if (collected.length >= target) break;
+    }
+
+    // Only pages spent crossing the newer band are free. A page that merely ran
+    // past the end of the window (older start-padding posts) is charged, since
+    // nothing further ahead can be in-window.
+    if (trimBandHours === 0 || pageTrimmedNewer === 0) pagesOnBudget += 1;
+
+    const step = advanceOrStop(payload, seenCursors);
+    if (typeof step === "string") {
+      stoppedBy = step;
+      break;
+    }
+    cursor = step.cursor;
+  }
+  if (collected.length >= target) stoppedBy = "target";
+
+  return {
+    query: params.query,
+    expression,
+    queryType: params.queryType ?? "Latest",
+    tweets: collected,
+    pagesFetched: pages,
+    window,
+    stoppedBy,
+    truncated: isTruncated(stoppedBy),
+    trimmedNewer: trimmedNewer > 0 ? trimmedNewer : undefined,
+    trimmedOlder: trimmedOlder > 0 ? trimmedOlder : undefined,
+  };
+}
+