zenrows 2.1.1 → 2.2.0

package/dist/index.js

@@ -5,7 +5,7 @@ import fetchRetry from "fetch-retry";
 // package.json
 var package_default = {
   name: "zenrows",
-  version: "2.1.1",
+  version: "2.2.0",
   description: "ZenRows Node SDK",
   main: "./dist/index.cjs",
   module: "./dist/index.js",
@@ -24,6 +24,7 @@ var package_default = {
   scripts: {
     build: "tsup src/index.ts --dts --format cjs,esm --clean",
     test: "vitest",
+    "test:e2e": "vitest run tests/e2e",
     format: "biome format --write .",
     check: "biome check ."
   },
@@ -175,12 +176,13 @@ var ZenRowsBatchError = class extends Error {
   code;
   extras;
   constructor(status, problem, extras) {
-    const message = problem ? `${status} ${problem.title ?? "Error"}: ${problem.detail ?? problem.code ?? "unknown"}` : `${status} (no problem body)`;
+    const code = problem?.code ?? "internal";
+    const message = problem ? `${status} ${problem.title ?? "Error"}: ${problem.detail ?? code}` : `${status} (no problem body)`;
     super(message);
     this.name = "ZenRowsBatchError";
     this.status = status;
-    this.problem = problem;
-    this.code = problem?.code ?? "internal";
+    this.problem = problem && { ...problem, code };
+    this.code = code;
     this.extras = extras;
   }
 };
@@ -206,7 +208,7 @@ async function parseProblem(response) {
     type: typeof record.type === "string" ? record.type : "about:blank",
     title: typeof record.title === "string" ? record.title : "Error",
     status: typeof record.status === "number" ? record.status : response.status,
-    code: typeof record.code === "string" ? record.code : "internal",
+    code: typeof record.code === "string" ? record.code : void 0,
     detail: typeof record.detail === "string" ? record.detail : void 0,
     instance: typeof record.instance === "string" ? record.instance : void 0
   };
@@ -629,6 +631,7 @@ var IDEMPOTENT_METHODS = /* @__PURE__ */ new Set(["GET", "PUT", "DELETE", "HEAD"
 var BACKOFF_BASE_MS = 250;
 var BACKOFF_CAP_MS = 1e4;
 var DEFAULT_RETRIES = 3;
+var RETRY_AFTER_CAP_MS = 3e4;
 function isIdempotent(method, hasIdempotencyKey) {
   return IDEMPOTENT_METHODS.has(method.toUpperCase()) || method.toUpperCase() === "POST" && hasIdempotencyKey;
 }
@@ -641,41 +644,71 @@ function retryAfterMs(response) {
   if (!raw) return void 0;
   const secs = Number(raw);
   if (Number.isNaN(secs) || secs < 0) return void 0;
-  return secs * 1e3;
+  return Math.min(secs * 1e3, RETRY_AFTER_CAP_MS);
 }
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
+var batchError = (response, problem, extras) => new ZenRowsBatchError(response.status, problem, extras);
+function deadline(ms) {
+  if (ms === void 0) return { clear() {
+  } };
+  const controller = new AbortController();
+  const timer = setTimeout(
+    () => controller.abort(new DOMException(`request timed out after ${ms / 1e3}s`, "TimeoutError")),
+    ms
+  );
+  return { signal: controller.signal, clear: () => clearTimeout(timer) };
+}
 var BatchTransport = class {
-  constructor(baseURL, apiKey, retries = DEFAULT_RETRIES) {
+  constructor(baseURL, apiKey, retries = DEFAULT_RETRIES, errorFactory = batchError, timeoutMs) {
     this.baseURL = baseURL;
     this.apiKey = apiKey;
     this.retries = retries;
+    this.errorFactory = errorFactory;
+    this.timeoutMs = timeoutMs;
   }
   baseURL;
   apiKey;
   retries;
-  /** Send a request, parse the response, throw `ZenRowsBatchError` on non-2xx. */
+  errorFactory;
+  timeoutMs;
+  /** Send a request, parse the response, throw the factory's error on non-2xx. */
   async requestJson(method, path, options = {}) {
-    const response = await this.send(method, path, options);
-    if (!response.ok) {
-      const { problem, extras } = await parseProblem(response);
-      throw new ZenRowsBatchError(response.status, problem, extras);
-    }
-    const text = await response.text();
+    const text = await this.requestText(method, path, options);
     if (!text) {
       return void 0;
     }
     return JSON.parse(text);
   }
-  /** Send a request and return the raw `Response` — used for endpoints that don't return JSON. */
+  /** Send a request and return its body as text; throw the factory's error on non-2xx. */
+  async requestText(method, path, options = {}) {
+    const { response, clear } = await this.send(method, path, options);
+    try {
+      await this.throwUnlessOk(response);
+      return await response.text();
+    } finally {
+      clear();
+    }
+  }
+  /**
+   * Send a request and return the raw `Response`, for bodies read as a stream. The timeout
+   * stops once the headers of a 2xx arrive; an error body is still read within it.
+   */
   async requestRaw(method, path, options = {}) {
-    const response = await this.send(method, path, options);
+    const { response, clear } = await this.send(method, path, options);
+    try {
+      await this.throwUnlessOk(response);
+    } finally {
+      clear();
+    }
+    return response;
+  }
+  async throwUnlessOk(response) {
     if (!response.ok) {
       const { problem, extras } = await parseProblem(response);
-      throw new ZenRowsBatchError(response.status, problem, extras);
+      throw this.errorFactory(response, problem, extras);
     }
-    return response;
   }
   buildUrl(path, query) {
     const url = new URL(`${this.baseURL}${path}`);
@@ -702,16 +735,20 @@ var BatchTransport = class {
       headers["Idempotency-Key"] = options.idempotencyKey;
     }
     const idempotent = isIdempotent(method, Boolean(options.idempotencyKey));
+    const retryStatuses = options.retryStatuses ?? RETRYABLE_STATUSES;
     let attempt = 0;
     while (true) {
+      const { signal, clear } = deadline(this.timeoutMs);
       let response;
       try {
         response = await fetch(url.toString(), {
           method,
           headers,
-          body: options.body !== void 0 ? JSON.stringify(options.body) : void 0
+          body: options.body !== void 0 ? JSON.stringify(options.body) : void 0,
+          signal
         });
       } catch (error) {
+        clear();
         if (idempotent && attempt < this.retries) {
           await sleep(backoffMs(attempt));
           attempt += 1;
@@ -719,13 +756,16 @@ var BatchTransport = class {
         }
         throw error;
       }
-      if (idempotent && attempt < this.retries && RETRYABLE_STATUSES.has(response.status)) {
+      if (idempotent && attempt < this.retries && retryStatuses.has(response.status)) {
+        response.body?.cancel().catch(() => {
+        });
+        clear();
         const wait = retryAfterMs(response) ?? backoffMs(attempt);
         await sleep(wait);
         attempt += 1;
         continue;
       }
-      return response;
+      return { response, clear };
     }
   }
 };
@@ -752,7 +792,7 @@ async function pollUntil(fetch2, {
   backoff = 1.5,
   jitter = 0.2
 }) {
-  const deadline = Date.now() + timeout * 1e3;
+  const deadline2 = Date.now() + timeout * 1e3;
   let interval = initialInterval;
   while (true) {
     const value = await fetch2();
@@ -765,10 +805,10 @@ async function pollUntil(fetch2, {
       );
     }
     const now = Date.now();
-    if (now >= deadline) {
+    if (now >= deadline2) {
       throw new WaiterTimeoutError(`waiter: timed out after ${timeout}s waiting for target state`);
     }
-    const remainingSeconds = (deadline - now) / 1e3;
+    const remainingSeconds = (deadline2 - now) / 1e3;
     const sleepSeconds = Math.min(interval, remainingSeconds) * (1 + jitter * (Math.random() * 2 - 1));
     if (sleepSeconds > 0) {
       await new Promise((resolve) => setTimeout(resolve, sleepSeconds * 1e3));
@@ -1166,6 +1206,220 @@ var ZenRowsBatchClient = class {
   }
 };
 
+// src/crawl/errors.ts
+var CRAWL_NOT_ENABLED_CODE = "REQS008";
+var ZenRowsCrawlError = class extends Error {
+  status;
+  problem;
+  code;
+  detail;
+  extras;
+  /** Seconds from the `Retry-After` header, e.g. 30 on a 429 `too_many_crawls`. */
+  retryAfter;
+  constructor(status, problem, extras, retryAfter) {
+    let message;
+    if (status === 403 && problem?.code === CRAWL_NOT_ENABLED_CODE) {
+      message = `403 Crawl is not enabled for this account (${CRAWL_NOT_ENABLED_CODE})${problem.detail ? `: ${problem.detail}` : ""}`;
+    } else if (problem) {
+      message = `${status} ${problem.title ?? "Error"}: ${problem.detail ?? problem.code ?? "unknown"}`;
+    } else {
+      message = `${status} (no problem body)`;
+    }
+    super(message);
+    this.name = "ZenRowsCrawlError";
+    this.status = status;
+    this.problem = problem;
+    this.code = problem?.code;
+    this.detail = problem?.detail;
+    this.extras = extras;
+    this.retryAfter = retryAfter;
+  }
+};
+function crawlError(response, problem, extras) {
+  const raw = response.headers.get("Retry-After");
+  const secs = raw === null || raw.trim() === "" ? Number.NaN : Number(raw);
+  return new ZenRowsCrawlError(
+    response.status,
+    problem,
+    extras,
+    Number.isNaN(secs) || secs < 0 ? void 0 : secs
+  );
+}
+
+// src/crawl/client.ts
+var DEFAULT_CRAWL_API_URL = "https://api.zenrows.com/v1";
+var DEFAULT_TIMEOUT_SECONDS = 30;
+var DEFAULT_WAIT_TIMEOUT_SECONDS = 600;
+var POLL_INTERVAL_SECONDS = 2;
+var MAX_POLL_INTERVAL_SECONDS = 15;
+var POLL_BACKOFF = 1.5;
+var CREATE_RETRY_STATUSES = /* @__PURE__ */ new Set([502, 503, 504]);
+var STOP_RETRY_STATUSES = /* @__PURE__ */ new Set([429, 502, 503, 504]);
+var MAX_STOP_RETRY_AFTER_SECONDS = 30;
+function contentIdOf(content) {
+  let ref;
+  if (typeof content === "string") ref = content;
+  else if (content.content_url) ref = content.content_url;
+  else {
+    throw new TypeError(
+      `content: result for ${content.url} has no content_url (content_status: ${content.content_status ?? "absent"})`
+    );
+  }
+  const id = ref.split("/").pop();
+  if (!id) throw new TypeError(`content: cannot read a content id from ${ref}`);
+  return id;
+}
+async function* ndjsonLines(body) {
+  if (!body) return;
+  const decoder = new TextDecoder();
+  let buffered = "";
+  for await (const chunk of body) {
+    buffered += decoder.decode(chunk, { stream: true });
+    let newline = buffered.indexOf("\n");
+    while (newline !== -1) {
+      const line = buffered.slice(0, newline).trim();
+      buffered = buffered.slice(newline + 1);
+      if (line) yield JSON.parse(line);
+      newline = buffered.indexOf("\n");
+    }
+  }
+  const last = (buffered + decoder.decode()).trim();
+  if (last) yield JSON.parse(last);
+}
+var ZenRowsCrawlClient = class {
+  apiKey;
+  transport;
+  retries;
+  constructor(apiKey, config = {}) {
+    this.apiKey = apiKey;
+    this.retries = config.retries ?? DEFAULT_RETRIES;
+    this.transport = new BatchTransport(
+      (config.baseURL ?? DEFAULT_CRAWL_API_URL).replace(/\/+$/, ""),
+      apiKey,
+      config.retries,
+      crawlError,
+      (config.timeout ?? DEFAULT_TIMEOUT_SECONDS) * 1e3
+    );
+  }
+  /**
+   * Start a crawl. Sends only the fields you set. With an `idempotencyKey`, a retry returns the
+   * crawl the first request created instead of starting another, and 5xx and network failures
+   * are retried. A 429 `too_many_crawls` is never retried: it throws `ZenRowsCrawlError` with
+   * `retryAfter` set.
+   */
+  create(params, opts = {}) {
+    const body = {
+      url: params.url,
+      depth: params.depth,
+      max_items: params.maxItems,
+      max_pages: params.maxPages,
+      include_patterns: params.includePatterns,
+      exclude_patterns: params.excludePatterns,
+      output_format: params.outputFormat
+    };
+    return this.transport.requestJson("POST", "/crawls", {
+      body,
+      idempotencyKey: opts.idempotencyKey,
+      retryStatuses: CREATE_RETRY_STATUSES
+    });
+  }
+  /** Read a crawl's status, coverage and one page of results (`limit` 1–10000, default 1000). */
+  get(crawlId, options = {}) {
+    return this.transport.requestJson("GET", `/crawls/${encodeURIComponent(crawlId)}`, {
+      query: options
+    });
+  }
+  /** One page of the account's crawls, newest first (`limit` 1–100, default 20). */
+  list(options = {}) {
+    return this.transport.requestJson("GET", "/crawls", { query: options });
+  }
+  /**
+   * Every URL the crawl kept, following `next_cursor` (`limit` is the page size). On a running
+   * crawl it yields the URLs kept so far and returns at the first empty page; it does not poll.
+   * Call `wait()` first to read every result.
+   */
+  async *results(crawlId, options = {}) {
+    let cursor;
+    while (true) {
+      const page = await this.get(crawlId, { ...options, cursor });
+      yield* page.results;
+      if (page.next_cursor === null || page.results.length === 0) return;
+      cursor = page.next_cursor;
+    }
+  }
+  /**
+   * Stop a running crawl. Idempotent: a crawl that already ended answers as it ended.
+   * Because a stop is safe to repeat, `stop()` retries a 429, 502, 503 (including `crawl_busy`,
+   * which means the stop was not saved and the crawl is still running) or 504 up to `retries`
+   * times, waiting `Retry-After` seconds (at most 30; 1 when the header is missing or unusable)
+   * each time, then throws the last error.
+   */
+  async stop(crawlId) {
+    for (let attempt = 0; ; attempt += 1) {
+      try {
+        return await this.transport.requestJson(
+          "POST",
+          `/crawls/${encodeURIComponent(crawlId)}/stop`
+        );
+      } catch (error) {
+        if (!(error instanceof ZenRowsCrawlError) || !STOP_RETRY_STATUSES.has(error.status) || attempt >= this.retries) {
+          throw error;
+        }
+        const waitSeconds = Math.min(error.retryAfter ?? 1, MAX_STOP_RETRY_AFTER_SECONDS);
+        await new Promise((resolve) => setTimeout(resolve, waitSeconds * 1e3));
+      }
+    }
+  }
+  /**
+   * One kept URL's page as fetched (HTML). Pass the content id, a `content_url`, or the
+   * `CrawlResult` itself (its `content_url` is present once `content_status` is `fetched`).
+   * Throws `TypeError` when no content id can be read.
+   */
+  async content(crawlId, contentId) {
+    return this.transport.requestText(
+      "GET",
+      `/crawls/${encodeURIComponent(crawlId)}/contents/${encodeURIComponent(contentIdOf(contentId))}`
+    );
+  }
+  /**
+   * Every result of the crawl in one download: its `status` and the parsed lines (with the
+   * page when the crawl has an `output_format`). On a running crawl, `status` is `running` and
+   * the lines hold only what was kept so far.
+   */
+  async download(crawlId) {
+    const response = await this.transport.requestRaw(
+      "GET",
+      `/crawls/${encodeURIComponent(crawlId)}/download`
+    );
+    return {
+      status: response.headers.get("X-Crawl-Status") ?? void 0,
+      lines: ndjsonLines(response.body)
+    };
+  }
+  /**
+   * Poll until the crawl ends (any status but `running`) or `timeout` seconds pass, and return
+   * it, without results. A `failed` crawl resolves too: read `error`. On timeout the crawl is
+   * returned with status `running` and keeps running.
+   */
+  async wait(crawlId, options = {}) {
+    const deadline2 = Date.now() + (options.timeout ?? DEFAULT_WAIT_TIMEOUT_SECONDS) * 1e3;
+    let interval = POLL_INTERVAL_SECONDS;
+    let latest;
+    while (true) {
+      latest = await this.get(crawlId, { limit: 1 });
+      const remaining = deadline2 - Date.now();
+      if (latest.status !== "running" || remaining <= 0) break;
+      const jitter = 0.8 + Math.random() * 0.4;
+      await new Promise(
+        (resolve) => setTimeout(resolve, Math.min(interval * 1e3 * jitter, remaining))
+      );
+      interval = Math.min(interval * POLL_BACKOFF, MAX_POLL_INTERVAL_SECONDS);
+    }
+    const { results: _results, next_cursor: _cursor, ...crawl } = latest;
+    return crawl;
+  }
+};
+
 // src/batch/schedule.ts
 var DAYS_OF_WEEK = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"];
 function validateTimezone(tz, field) {
@@ -1335,10 +1589,13 @@ var ZenRows = class {
   fetchWithRetry;
   /** Client for the Batch API (async job/run/task model) — see `./batch.ts`. */
   batch;
+  /** Client for the Crawl API (new; one start URL in, the URLs behind it out). See `./crawl/client.ts`. */
+  crawl;
   constructor(apiKey, clientConfig = {}) {
     this.apiKey = apiKey;
     this.clientConfig = clientConfig;
     this.batch = new ZenRowsBatchClient(apiKey);
+    this.crawl = new ZenRowsCrawlClient(apiKey, clientConfig.crawl);
     const retries = this.clientConfig.retries ?? 0;
     this.queue = fastq.promise(this, this.worker, this.clientConfig.concurrency ?? 5);
     this.fetchWithRetry = fetchRetry(fetch, {
@@ -1469,6 +1726,7 @@ export {
   AUTO_MIN_CREDITS,
   At,
   BASE_CREDITS,
+  CRAWL_NOT_ENABLED_CODE,
   Calendar,
   CurrentRun,
   DEFAULT_MAX_BYTES_PER_FILE,
@@ -1493,6 +1751,8 @@ export {
   ZenRows,
   ZenRowsBatchClient,
   ZenRowsBatchError,
+  ZenRowsCrawlClient,
+  ZenRowsCrawlError,
   downloadTaskToFile,
   downloadTaskToMemory,
   downloadToDir,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zenrows",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "description": "ZenRows Node SDK",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -19,6 +19,7 @@
   "scripts": {
     "build": "tsup src/index.ts --dts --format cjs,esm --clean",
     "test": "vitest",
+    "test:e2e": "vitest run tests/e2e",
     "format": "biome format --write .",
     "check": "biome check ."
   },