zenrows 2.1.1 → 2.2.0

package/README.md

@@ -19,10 +19,12 @@ SDK to access [Zenrows](https://www.zenrows.com/) API directly from Node.js. Zen
   - [Batch](#batch)
     - [Failed runs and extensible values](#failed-runs-and-extensible-values)
     - [Extract in a batch](#extract-in-a-batch)
+  - [Crawl](#crawl)
   - [Concurrency](#concurrency)
     - [An important note about Promise.allSettled() on TypeScript](#an-important-note-about-promiseallsettled-on-typescript)
 - [Examples](#examples)
 - [Contributing](#contributing)
+  - [Running the Crawl e2e test](#running-the-crawl-e2e-test)
 - [License](#license)
 
 ## Installation
@@ -265,6 +267,77 @@ const { ZenRowsBatchClient } = require("zenrows");
 const batch = new ZenRowsBatchClient(apiKey, { baseURL: "https://async.api.zenrows.com/v1" }); // baseURL is optional
 ```
 
+### Crawl
+
+[Crawl](https://docs.zenrows.com/crawl/introduction) (new) takes one start URL and returns the URLs behind it, and is reachable via `client.crawl`. It follows the links on each page up to `depth` hops; without `depth` it has no depth limit and follows links until `maxPages` or `maxItems` stops it (each defaults to 100), or no link is left to open. A crawl stays on the start URL's registrable domain; subdomains count. A crawl runs asynchronously: `create()` returns at once, `wait()` polls until it ends, and `results()` reads the URLs it kept. With `outputFormat: "html"` it also fetches each kept page, which you read with `content()` or all at once with `download()`.
+
+Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change.
+
+```javascript
+const { ZenRows } = require("zenrows");
+
+const apiKey = "YOUR-API-KEY";
+
+(async () => {
+  const client = new ZenRows(apiKey);
+
+  const crawl = await client.crawl.create({
+    url: "https://www.scrapingcourse.com/ecommerce/",
+    depth: 1, // link hops from the start URL; omit for no depth limit
+    maxItems: 10, // stop once this many URLs are kept
+    maxPages: 20, // stop once this many pages are fetched; bounds the cost
+    includePatterns: ["/product/"], // keep only URLs containing one of these
+    outputFormat: "html", // omit for URLs only
+  });
+
+  const ended = await client.crawl.wait(crawl.crawl_id); // completed, stopped, failed, or running on timeout
+  if (ended.status === "failed") throw new Error(ended.error?.detail);
+
+  for await (const result of client.crawl.results(crawl.crawl_id)) {
+    console.log(result.url);
+    if (result.content_status === "fetched") {
+      const html = await client.crawl.content(crawl.crawl_id, result);
+    }
+  }
+
+  // Or every result, with its page, as parsed NDJSON lines.
+  const download = await client.crawl.download(crawl.crawl_id);
+  console.log(download.status); // from X-Crawl-Status; "running" means more lines may come later
+  for await (const line of download.lines) {
+    console.log(line.url, line.content);
+  }
+})();
+```
+
+`client.crawl` also exposes `get()` (status, coverage and one page of results), `list()` (one page of the account's crawls, newest first, with `next_cursor` until the last page), and `stop()` (idempotent; a crawl that already ended answers as it ended). `create()` takes an `idempotencyKey` option, so a retried create does not start a second crawl.
+
+`results()` does not poll. On a running crawl it yields only what the crawl has kept so far, then returns. Call `wait()` first to read every result. `wait()` polls for up to `timeout` seconds (default 600). When the time runs out, it returns the crawl with status `running` and does not throw; the crawl keeps running.
+
+Errors throw a `ZenRowsCrawlError` with `status`, `code`, `detail` and `retryAfter`. `code` is undefined when the error body has no code. Branch on `status` and `code`:
+
+- `400` `invalid_request` / `unknown_parameter` / `invalid_cursor`.
+- `403` `REQS008`: Crawl is not enabled for this account.
+- `404` `crawl_not_found` / `content_not_found`.
+- `409` `idempotency_request_in_flight`: a request with the same `idempotencyKey` is still running. Retry after it finishes.
+- `422` `invalid_parameter` / `invalid_start_url` / `domain_not_allowed`: `detail` names the problem. `422` `idempotency_key_reused`: the key was used with a different body. Use a new key or no key; do not retry as is.
+- `429` `too_many_crawls`: the account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Nothing was created. `create()` never retries it; retry after `error.retryAfter` seconds.
+- `503` `crawl_busy`: the stop could not be saved yet; the crawl is still running. Repeating the stop is safe, so `stop()` retries it, and any other 429, 502, 503 or 504, up to `retries` times, waiting `error.retryAfter` seconds (at most 30, or 1 when the header is missing), then throws the last error.
+
+`content()` takes a content id, a `content_url` or a result, and throws a `TypeError` when it cannot read a content id from it. Requests retry network errors and 502/503/504 (and 429 on reads) when it is safe; a `create()` is retried only with an `idempotencyKey`, and a `stop()` as described above.
+
+`outputFormat` takes `"html"`; without it a crawl returns URLs only. `maxPages` and `maxItems` default to 100 each. A crawl created without `depth` has no `depth` field; one created with it echoes it. Response enum fields (`status`, `stop_reason`, `error.code`, `content_status`) are `Extensible<...>`: the known values plus any `string`. A failed crawl carries `error.code` and `error.detail`. A download line's `content` is a string, or an object for a crawl that returns JSON.
+
+The crawl client (`ZenRowsCrawlClient`) also works standalone. Both forms take the same options:
+
+```javascript
+const { ZenRows, ZenRowsCrawlClient } = require("zenrows");
+
+// All options are optional. timeout is in seconds and applies to each HTTP request, not to reading a download.
+const options = { baseURL: "https://api.zenrows.com/v1", retries: 3, timeout: 30 };
+const crawl = new ZenRowsCrawlClient(apiKey, options);
+const client = new ZenRows(apiKey, { crawl: options }); // client.crawl uses them
+```
+
 ### Concurrency
 
 To limit the concurrency, it uses [fastq](https://github.com/mcollina/fastq), which will simultaneously send a maximum of requests. The concurrency is determined by the plan you are in, so take a look at the [pricing](https://www.zenrows.com/pricing) and set it accordingly. Take into account that each client instance will have its own limit, meaning that two different scripts will not share it, and 429 (Too Many Requests) errors might arise.
@@ -340,6 +413,26 @@ npx tsx index.ts # TS example
 
 Pull requests are welcome. For significant changes, please open an issue first to discuss what you would like to change.
 
+Before opening a pull request, run `pnpm check` and `pnpm test -- --run`. Neither touches the network.
+
+### Running the Crawl e2e test
+
+`tests/e2e/crawl.e2e.test.ts` runs a small real crawl (depth 1, up to 3 items and 5 pages, HTML output) and checks every Crawl method against the live API. It is skipped unless `ZENROWS_API_KEY` and `ZENROWS_E2E_CRAWL_URL` are set:
+
+- `ZENROWS_API_KEY`: a key with Crawl access. Keep it out of your shell history, e.g. read it from a file.
+- `ZENROWS_CRAWL_BASE_URL` (optional): the API base. Default `https://api.zenrows.com/v1`. Point it at a local or staging deployment to test that instead.
+- `ZENROWS_E2E_CRAWL_URL`: the start URL to crawl, a site you may crawl whose start page links to at least one page.
+- `ZENROWS_E2E_CRAWL_INCLUDE` (optional): an include pattern, e.g. `/product/`. When set, the crawl keeps only matching URLs and the test asserts every result contains it.
+
+```bash
+export ZENROWS_API_KEY="$(cat path/to/key-file)"
+export ZENROWS_E2E_CRAWL_URL=https://www.scrapingcourse.com/ecommerce/
+export ZENROWS_E2E_CRAWL_INCLUDE=/product/
+pnpm test:e2e
+```
+
+The run creates one crawl on the account and uses up to a few pages of credits. If the account has reached its limit of active jobs, the test waits and retries (up to 5 minutes).
+
 ## License
 
 [MIT](./LICENSE)

package/dist/index.cjs

@@ -34,6 +34,7 @@ __export(index_exports, {
   AUTO_MIN_CREDITS: () => AUTO_MIN_CREDITS,
   At: () => At,
   BASE_CREDITS: () => BASE_CREDITS,
+  CRAWL_NOT_ENABLED_CODE: () => CRAWL_NOT_ENABLED_CODE,
   Calendar: () => Calendar,
   CurrentRun: () => CurrentRun,
   DEFAULT_MAX_BYTES_PER_FILE: () => DEFAULT_MAX_BYTES_PER_FILE,
@@ -58,6 +59,8 @@ __export(index_exports, {
   ZenRows: () => ZenRows,
   ZenRowsBatchClient: () => ZenRowsBatchClient,
   ZenRowsBatchError: () => ZenRowsBatchError,
+  ZenRowsCrawlClient: () => ZenRowsCrawlClient,
+  ZenRowsCrawlError: () => ZenRowsCrawlError,
   downloadTaskToFile: () => downloadTaskToFile,
   downloadTaskToMemory: () => downloadTaskToMemory,
   downloadToDir: () => downloadToDir,
@@ -73,7 +76,7 @@ var import_fetch_retry = __toESM(require("fetch-retry"), 1);
 // package.json
 var package_default = {
   name: "zenrows",
-  version: "2.1.1",
+  version: "2.2.0",
   description: "ZenRows Node SDK",
   main: "./dist/index.cjs",
   module: "./dist/index.js",
@@ -92,6 +95,7 @@ var package_default = {
   scripts: {
     build: "tsup src/index.ts --dts --format cjs,esm --clean",
     test: "vitest",
+    "test:e2e": "vitest run tests/e2e",
     format: "biome format --write .",
     check: "biome check ."
   },
@@ -243,12 +247,13 @@ var ZenRowsBatchError = class extends Error {
   code;
   extras;
   constructor(status, problem, extras) {
-    const message = problem ? `${status} ${problem.title ?? "Error"}: ${problem.detail ?? problem.code ?? "unknown"}` : `${status} (no problem body)`;
+    const code = problem?.code ?? "internal";
+    const message = problem ? `${status} ${problem.title ?? "Error"}: ${problem.detail ?? code}` : `${status} (no problem body)`;
     super(message);
     this.name = "ZenRowsBatchError";
     this.status = status;
-    this.problem = problem;
-    this.code = problem?.code ?? "internal";
+    this.problem = problem && { ...problem, code };
+    this.code = code;
     this.extras = extras;
   }
 };
@@ -274,7 +279,7 @@ async function parseProblem(response) {
     type: typeof record.type === "string" ? record.type : "about:blank",
     title: typeof record.title === "string" ? record.title : "Error",
     status: typeof record.status === "number" ? record.status : response.status,
-    code: typeof record.code === "string" ? record.code : "internal",
+    code: typeof record.code === "string" ? record.code : void 0,
     detail: typeof record.detail === "string" ? record.detail : void 0,
     instance: typeof record.instance === "string" ? record.instance : void 0
   };
@@ -697,6 +702,7 @@ var IDEMPOTENT_METHODS = /* @__PURE__ */ new Set(["GET", "PUT", "DELETE", "HEAD"
 var BACKOFF_BASE_MS = 250;
 var BACKOFF_CAP_MS = 1e4;
 var DEFAULT_RETRIES = 3;
+var RETRY_AFTER_CAP_MS = 3e4;
 function isIdempotent(method, hasIdempotencyKey) {
   return IDEMPOTENT_METHODS.has(method.toUpperCase()) || method.toUpperCase() === "POST" && hasIdempotencyKey;
 }
@@ -709,41 +715,71 @@ function retryAfterMs(response) {
   if (!raw) return void 0;
   const secs = Number(raw);
   if (Number.isNaN(secs) || secs < 0) return void 0;
-  return secs * 1e3;
+  return Math.min(secs * 1e3, RETRY_AFTER_CAP_MS);
 }
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
+var batchError = (response, problem, extras) => new ZenRowsBatchError(response.status, problem, extras);
+function deadline(ms) {
+  if (ms === void 0) return { clear() {
+  } };
+  const controller = new AbortController();
+  const timer = setTimeout(
+    () => controller.abort(new DOMException(`request timed out after ${ms / 1e3}s`, "TimeoutError")),
+    ms
+  );
+  return { signal: controller.signal, clear: () => clearTimeout(timer) };
+}
 var BatchTransport = class {
-  constructor(baseURL, apiKey, retries = DEFAULT_RETRIES) {
+  constructor(baseURL, apiKey, retries = DEFAULT_RETRIES, errorFactory = batchError, timeoutMs) {
     this.baseURL = baseURL;
     this.apiKey = apiKey;
     this.retries = retries;
+    this.errorFactory = errorFactory;
+    this.timeoutMs = timeoutMs;
   }
   baseURL;
   apiKey;
   retries;
-  /** Send a request, parse the response, throw `ZenRowsBatchError` on non-2xx. */
+  errorFactory;
+  timeoutMs;
+  /** Send a request, parse the response, throw the factory's error on non-2xx. */
   async requestJson(method, path, options = {}) {
-    const response = await this.send(method, path, options);
-    if (!response.ok) {
-      const { problem, extras } = await parseProblem(response);
-      throw new ZenRowsBatchError(response.status, problem, extras);
-    }
-    const text = await response.text();
+    const text = await this.requestText(method, path, options);
     if (!text) {
       return void 0;
     }
     return JSON.parse(text);
   }
-  /** Send a request and return the raw `Response` — used for endpoints that don't return JSON. */
+  /** Send a request and return its body as text; throw the factory's error on non-2xx. */
+  async requestText(method, path, options = {}) {
+    const { response, clear } = await this.send(method, path, options);
+    try {
+      await this.throwUnlessOk(response);
+      return await response.text();
+    } finally {
+      clear();
+    }
+  }
+  /**
+   * Send a request and return the raw `Response`, for bodies read as a stream. The timeout
+   * stops once the headers of a 2xx arrive; an error body is still read within it.
+   */
   async requestRaw(method, path, options = {}) {
-    const response = await this.send(method, path, options);
+    const { response, clear } = await this.send(method, path, options);
+    try {
+      await this.throwUnlessOk(response);
+    } finally {
+      clear();
+    }
+    return response;
+  }
+  async throwUnlessOk(response) {
     if (!response.ok) {
       const { problem, extras } = await parseProblem(response);
-      throw new ZenRowsBatchError(response.status, problem, extras);
+      throw this.errorFactory(response, problem, extras);
     }
-    return response;
   }
   buildUrl(path, query) {
     const url = new URL(`${this.baseURL}${path}`);
@@ -770,16 +806,20 @@ var BatchTransport = class {
       headers["Idempotency-Key"] = options.idempotencyKey;
     }
     const idempotent = isIdempotent(method, Boolean(options.idempotencyKey));
+    const retryStatuses = options.retryStatuses ?? RETRYABLE_STATUSES;
     let attempt = 0;
     while (true) {
+      const { signal, clear } = deadline(this.timeoutMs);
       let response;
       try {
         response = await fetch(url.toString(), {
           method,
           headers,
-          body: options.body !== void 0 ? JSON.stringify(options.body) : void 0
+          body: options.body !== void 0 ? JSON.stringify(options.body) : void 0,
+          signal
         });
       } catch (error) {
+        clear();
         if (idempotent && attempt < this.retries) {
           await sleep(backoffMs(attempt));
           attempt += 1;
@@ -787,13 +827,16 @@ var BatchTransport = class {
         }
         throw error;
       }
-      if (idempotent && attempt < this.retries && RETRYABLE_STATUSES.has(response.status)) {
+      if (idempotent && attempt < this.retries && retryStatuses.has(response.status)) {
+        response.body?.cancel().catch(() => {
+        });
+        clear();
         const wait = retryAfterMs(response) ?? backoffMs(attempt);
         await sleep(wait);
         attempt += 1;
         continue;
       }
-      return response;
+      return { response, clear };
     }
   }
 };
@@ -820,7 +863,7 @@ async function pollUntil(fetch2, {
   backoff = 1.5,
   jitter = 0.2
 }) {
-  const deadline = Date.now() + timeout * 1e3;
+  const deadline2 = Date.now() + timeout * 1e3;
   let interval = initialInterval;
   while (true) {
     const value = await fetch2();
@@ -833,10 +876,10 @@ async function pollUntil(fetch2, {
       );
     }
     const now = Date.now();
-    if (now >= deadline) {
+    if (now >= deadline2) {
       throw new WaiterTimeoutError(`waiter: timed out after ${timeout}s waiting for target state`);
     }
-    const remainingSeconds = (deadline - now) / 1e3;
+    const remainingSeconds = (deadline2 - now) / 1e3;
     const sleepSeconds = Math.min(interval, remainingSeconds) * (1 + jitter * (Math.random() * 2 - 1));
     if (sleepSeconds > 0) {
       await new Promise((resolve) => setTimeout(resolve, sleepSeconds * 1e3));
@@ -1234,6 +1277,220 @@ var ZenRowsBatchClient = class {
   }
 };
 
+// src/crawl/errors.ts
+var CRAWL_NOT_ENABLED_CODE = "REQS008";
+var ZenRowsCrawlError = class extends Error {
+  status;
+  problem;
+  code;
+  detail;
+  extras;
+  /** Seconds from the `Retry-After` header, e.g. 30 on a 429 `too_many_crawls`. */
+  retryAfter;
+  constructor(status, problem, extras, retryAfter) {
+    let message;
+    if (status === 403 && problem?.code === CRAWL_NOT_ENABLED_CODE) {
+      message = `403 Crawl is not enabled for this account (${CRAWL_NOT_ENABLED_CODE})${problem.detail ? `: ${problem.detail}` : ""}`;
+    } else if (problem) {
+      message = `${status} ${problem.title ?? "Error"}: ${problem.detail ?? problem.code ?? "unknown"}`;
+    } else {
+      message = `${status} (no problem body)`;
+    }
+    super(message);
+    this.name = "ZenRowsCrawlError";
+    this.status = status;
+    this.problem = problem;
+    this.code = problem?.code;
+    this.detail = problem?.detail;
+    this.extras = extras;
+    this.retryAfter = retryAfter;
+  }
+};
+function crawlError(response, problem, extras) {
+  const raw = response.headers.get("Retry-After");
+  const secs = raw === null || raw.trim() === "" ? Number.NaN : Number(raw);
+  return new ZenRowsCrawlError(
+    response.status,
+    problem,
+    extras,
+    Number.isNaN(secs) || secs < 0 ? void 0 : secs
+  );
+}
+
+// src/crawl/client.ts
+var DEFAULT_CRAWL_API_URL = "https://api.zenrows.com/v1";
+var DEFAULT_TIMEOUT_SECONDS = 30;
+var DEFAULT_WAIT_TIMEOUT_SECONDS = 600;
+var POLL_INTERVAL_SECONDS = 2;
+var MAX_POLL_INTERVAL_SECONDS = 15;
+var POLL_BACKOFF = 1.5;
+var CREATE_RETRY_STATUSES = /* @__PURE__ */ new Set([502, 503, 504]);
+var STOP_RETRY_STATUSES = /* @__PURE__ */ new Set([429, 502, 503, 504]);
+var MAX_STOP_RETRY_AFTER_SECONDS = 30;
+function contentIdOf(content) {
+  let ref;
+  if (typeof content === "string") ref = content;
+  else if (content.content_url) ref = content.content_url;
+  else {
+    throw new TypeError(
+      `content: result for ${content.url} has no content_url (content_status: ${content.content_status ?? "absent"})`
+    );
+  }
+  const id = ref.split("/").pop();
+  if (!id) throw new TypeError(`content: cannot read a content id from ${ref}`);
+  return id;
+}
+async function* ndjsonLines(body) {
+  if (!body) return;
+  const decoder = new TextDecoder();
+  let buffered = "";
+  for await (const chunk of body) {
+    buffered += decoder.decode(chunk, { stream: true });
+    let newline = buffered.indexOf("\n");
+    while (newline !== -1) {
+      const line = buffered.slice(0, newline).trim();
+      buffered = buffered.slice(newline + 1);
+      if (line) yield JSON.parse(line);
+      newline = buffered.indexOf("\n");
+    }
+  }
+  const last = (buffered + decoder.decode()).trim();
+  if (last) yield JSON.parse(last);
+}
+var ZenRowsCrawlClient = class {
+  apiKey;
+  transport;
+  retries;
+  constructor(apiKey, config = {}) {
+    this.apiKey = apiKey;
+    this.retries = config.retries ?? DEFAULT_RETRIES;
+    this.transport = new BatchTransport(
+      (config.baseURL ?? DEFAULT_CRAWL_API_URL).replace(/\/+$/, ""),
+      apiKey,
+      config.retries,
+      crawlError,
+      (config.timeout ?? DEFAULT_TIMEOUT_SECONDS) * 1e3
+    );
+  }
+  /**
+   * Start a crawl. Sends only the fields you set. With an `idempotencyKey`, a retry returns the
+   * crawl the first request created instead of starting another, and 5xx and network failures
+   * are retried. A 429 `too_many_crawls` is never retried: it throws `ZenRowsCrawlError` with
+   * `retryAfter` set.
+   */
+  create(params, opts = {}) {
+    const body = {
+      url: params.url,
+      depth: params.depth,
+      max_items: params.maxItems,
+      max_pages: params.maxPages,
+      include_patterns: params.includePatterns,
+      exclude_patterns: params.excludePatterns,
+      output_format: params.outputFormat
+    };
+    return this.transport.requestJson("POST", "/crawls", {
+      body,
+      idempotencyKey: opts.idempotencyKey,
+      retryStatuses: CREATE_RETRY_STATUSES
+    });
+  }
+  /** Read a crawl's status, coverage and one page of results (`limit` 1–10000, default 1000). */
+  get(crawlId, options = {}) {
+    return this.transport.requestJson("GET", `/crawls/${encodeURIComponent(crawlId)}`, {
+      query: options
+    });
+  }
+  /** One page of the account's crawls, newest first (`limit` 1–100, default 20). */
+  list(options = {}) {
+    return this.transport.requestJson("GET", "/crawls", { query: options });
+  }
+  /**
+   * Every URL the crawl kept, following `next_cursor` (`limit` is the page size). On a running
+   * crawl it yields the URLs kept so far and returns at the first empty page; it does not poll.
+   * Call `wait()` first to read every result.
+   */
+  async *results(crawlId, options = {}) {
+    let cursor;
+    while (true) {
+      const page = await this.get(crawlId, { ...options, cursor });
+      yield* page.results;
+      if (page.next_cursor === null || page.results.length === 0) return;
+      cursor = page.next_cursor;
+    }
+  }
+  /**
+   * Stop a running crawl. Idempotent: a crawl that already ended answers as it ended.
+   * Because a stop is safe to repeat, `stop()` retries a 429, 502, 503 (including `crawl_busy`,
+   * which means the stop was not saved and the crawl is still running) or 504 up to `retries`
+   * times, waiting `Retry-After` seconds (at most 30; 1 when the header is missing or unusable)
+   * each time, then throws the last error.
+   */
+  async stop(crawlId) {
+    for (let attempt = 0; ; attempt += 1) {
+      try {
+        return await this.transport.requestJson(
+          "POST",
+          `/crawls/${encodeURIComponent(crawlId)}/stop`
+        );
+      } catch (error) {
+        if (!(error instanceof ZenRowsCrawlError) || !STOP_RETRY_STATUSES.has(error.status) || attempt >= this.retries) {
+          throw error;
+        }
+        const waitSeconds = Math.min(error.retryAfter ?? 1, MAX_STOP_RETRY_AFTER_SECONDS);
+        await new Promise((resolve) => setTimeout(resolve, waitSeconds * 1e3));
+      }
+    }
+  }
+  /**
+   * One kept URL's page as fetched (HTML). Pass the content id, a `content_url`, or the
+   * `CrawlResult` itself (its `content_url` is present once `content_status` is `fetched`).
+   * Throws `TypeError` when no content id can be read.
+   */
+  async content(crawlId, contentId) {
+    return this.transport.requestText(
+      "GET",
+      `/crawls/${encodeURIComponent(crawlId)}/contents/${encodeURIComponent(contentIdOf(contentId))}`
+    );
+  }
+  /**
+   * Every result of the crawl in one download: its `status` and the parsed lines (with the
+   * page when the crawl has an `output_format`). On a running crawl, `status` is `running` and
+   * the lines hold only what was kept so far.
+   */
+  async download(crawlId) {
+    const response = await this.transport.requestRaw(
+      "GET",
+      `/crawls/${encodeURIComponent(crawlId)}/download`
+    );
+    return {
+      status: response.headers.get("X-Crawl-Status") ?? void 0,
+      lines: ndjsonLines(response.body)
+    };
+  }
+  /**
+   * Poll until the crawl ends (any status but `running`) or `timeout` seconds pass, and return
+   * it, without results. A `failed` crawl resolves too: read `error`. On timeout the crawl is
+   * returned with status `running` and keeps running.
+   */
+  async wait(crawlId, options = {}) {
+    const deadline2 = Date.now() + (options.timeout ?? DEFAULT_WAIT_TIMEOUT_SECONDS) * 1e3;
+    let interval = POLL_INTERVAL_SECONDS;
+    let latest;
+    while (true) {
+      latest = await this.get(crawlId, { limit: 1 });
+      const remaining = deadline2 - Date.now();
+      if (latest.status !== "running" || remaining <= 0) break;
+      const jitter = 0.8 + Math.random() * 0.4;
+      await new Promise(
+        (resolve) => setTimeout(resolve, Math.min(interval * 1e3 * jitter, remaining))
+      );
+      interval = Math.min(interval * POLL_BACKOFF, MAX_POLL_INTERVAL_SECONDS);
+    }
+    const { results: _results, next_cursor: _cursor, ...crawl } = latest;
+    return crawl;
+  }
+};
+
 // src/batch/schedule.ts
 var DAYS_OF_WEEK = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"];
 function validateTimezone(tz, field) {
@@ -1403,10 +1660,13 @@ var ZenRows = class {
   fetchWithRetry;
   /** Client for the Batch API (async job/run/task model) — see `./batch.ts`. */
   batch;
+  /** Client for the Crawl API (new; one start URL in, the URLs behind it out). See `./crawl/client.ts`. */
+  crawl;
   constructor(apiKey, clientConfig = {}) {
     this.apiKey = apiKey;
     this.clientConfig = clientConfig;
     this.batch = new ZenRowsBatchClient(apiKey);
+    this.crawl = new ZenRowsCrawlClient(apiKey, clientConfig.crawl);
     const retries = this.clientConfig.retries ?? 0;
     this.queue = import_fastq.default.promise(this, this.worker, this.clientConfig.concurrency ?? 5);
     this.fetchWithRetry = (0, import_fetch_retry.default)(fetch, {
@@ -1538,6 +1798,7 @@ var ZenRows = class {
   AUTO_MIN_CREDITS,
   At,
   BASE_CREDITS,
+  CRAWL_NOT_ENABLED_CODE,
   Calendar,
   CurrentRun,
   DEFAULT_MAX_BYTES_PER_FILE,
@@ -1562,6 +1823,8 @@ var ZenRows = class {
   ZenRows,
   ZenRowsBatchClient,
   ZenRowsBatchError,
+  ZenRowsCrawlClient,
+  ZenRowsCrawlError,
   downloadTaskToFile,
   downloadTaskToMemory,
   downloadToDir,