zenrows 2.0.3 → 2.2.0

package/dist/index.d.ts

@@ -1,14 +1,29 @@
 import * as fetch_retry from 'fetch-retry';
 import fastq from 'fastq';
 
+/**
+ * A server-extensible string enum: the listed values autocomplete, but the server may
+ * return values not listed (the Batch API marks these `x-extensible-enum`). Response fields
+ * use `Extensible<...>`, so callers must handle an unknown value. The named types stay closed
+ * so they keep working as request input.
+ */
+type Extensible<T extends string> = T | (string & {});
 type JobType = "regular" | "scheduled";
 type JobStatus = "open" | "closed" | "deleted";
 type ScheduleState = "active" | "paused";
 type RunStatus = "pending" | "running" | "completed" | "failed" | "stopped" | "deleted";
-type TaskStatus = "pending" | "successful" | "failed";
+type TaskStatus = "pending" | "processing" | "successful" | "failed";
 type PauseState = "active" | "paused";
 type IngestStatus = "pending" | "done";
 type ExportStatus = "pending" | "running" | "completed" | "failed";
+/**
+ * Why a run auto-failed (`status == "failed"`).
+ * - `insufficient_credits` — the account is out of credits.
+ * - `subscription_inactive` — the subscription is not active.
+ * - `api_key_cap_reached` — the job's API key reached one of its credit caps; resolves when
+ *   that cap's window resets or the cap is raised. `Run.failure_detail` says which.
+ */
+type RunFailureReason = "insufficient_credits" | "subscription_inactive" | "api_key_cap_reached";
 interface Spend {
     credits: number;
     cost: number;
@@ -53,19 +68,21 @@ interface Run {
     run_id: string;
     job_id: string;
     run_sequence: number;
-    status: RunStatus;
+    status: Extensible<RunStatus>;
     stats: RunStats;
     last_batch_received?: boolean;
     pause_state?: PauseState;
-    ingest_status?: IngestStatus;
-    failure_reason?: "insufficient_credits" | "subscription_inactive";
+    ingest_status?: Extensible<IngestStatus>;
+    failure_reason?: Extensible<RunFailureReason>;
+    /** Human-readable explanation of `failure_reason` (e.g. which cap, and when it resets, UTC). */
+    failure_detail?: string | null;
     created_at?: string;
     updated_at?: string;
 }
 interface Job {
     job_id: string;
     type: JobType;
-    status: JobStatus;
+    status: Extensible<JobStatus>;
     format?: string;
     zenrows_params?: Record<string, string>;
     external_id?: string;
@@ -89,19 +106,19 @@ interface ListJobRunsResponse {
 }
 interface SubmitJobResponse {
     job_id: string;
-    status: JobStatus;
+    status: Extensible<JobStatus>;
     latest_run?: Run;
     accepted_tasks: number;
     webhook?: WebhookConfig;
 }
 interface AddTasksResponse {
     accepted_tasks: number;
-    job_status: JobStatus;
+    job_status: Extensible<JobStatus>;
     latest_run: Run;
 }
 interface RerunJobResponse {
     job_id: string;
-    status: JobStatus;
+    status: Extensible<JobStatus>;
     latest_run: Run;
     rerun_of?: string;
     retried_tasks: number;
@@ -122,7 +139,7 @@ interface TaskResult {
     url: string;
     metadata?: Record<string, string>;
     method?: "GET" | "POST";
-    status: TaskStatus;
+    status: Extensible<TaskStatus>;
     type?: string;
     result_url?: string;
     error?: ProblemJson;
@@ -145,13 +162,13 @@ interface TaskHistoryResponse {
 }
 interface StartExportResponse {
     export_id: string;
-    status: ExportStatus;
+    status: Extensible<ExportStatus>;
     created_at: string;
     expires_at: string;
 }
 interface Export {
     export_id: string;
-    status: ExportStatus;
+    status: Extensible<ExportStatus>;
     error?: string;
     download_url?: string;
     created_at: string;
@@ -350,7 +367,7 @@ declare class RunRef {
 declare class RunHandle extends RunRef {
     readonly data: Run;
     constructor(client: ZenRowsBatchClient, jobId: string, runId: string, data: Run);
-    get status(): RunStatus;
+    get status(): Extensible<RunStatus>;
     get stats(): RunStats;
 }
 declare class ExportRef {
@@ -372,7 +389,7 @@ declare class ExportRef {
 declare class ExportHandle extends ExportRef {
     readonly data: Export;
     constructor(client: ZenRowsBatchClient, jobId: string, runId: string, exportId: string, data: Export, startResponse?: StartExportResponse);
-    get status(): ExportStatus;
+    get status(): Extensible<ExportStatus>;
 }
 /** Operations on a job's CURRENT run, reached via `job.run`. */
 declare class CurrentRun {
@@ -431,7 +448,7 @@ declare class JobRef {
     /** Operations on the schedule — pause/resume/update (scheduled jobs only). */
     get schedule(): ScheduleControls;
     /** The job status from the submit response — only known on refs from a `submit*` call. */
-    get status(): JobStatus | undefined;
+    get status(): Extensible<JobStatus> | undefined;
     get acceptedTasks(): number | undefined;
     load(): Promise<JobHandle>;
     close(): Promise<JobHandle>;
@@ -449,7 +466,7 @@ declare class JobRef {
         lastBatch?: boolean;
     }): Promise<{
         accepted_tasks: number;
-        job_status: JobStatus;
+        job_status: Extensible<JobStatus>;
         latest_run: Run;
     }>;
     runs(pageSize?: number): AsyncGenerator<RunHandle>;
@@ -475,7 +492,7 @@ declare class JobRef {
 declare class JobHandle extends JobRef {
     readonly data: Job;
     constructor(client: ZenRowsBatchClient, jobId: string, data: Job, submitResponse?: SubmitJobResponse);
-    get status(): JobStatus;
+    get status(): Extensible<JobStatus>;
 }
 
 interface BatchClientConfig {
@@ -531,7 +548,16 @@ interface GetResultsOptions {
 }
 interface WaitForRunOptions {
     runId?: string;
+    /**
+     * Statuses that resolve the wait. Defaults to every terminal status (`completed`,
+     * `stopped`, `failed`, `deleted`), so an auto-failed run resolves with its
+     * `failure_reason` / `failure_detail` instead of polling until `timeout`.
+     */
     targetStatuses?: Set<string>;
+    /**
+     * Statuses that reject with `WaiterFailureError`. Takes precedence over `targetStatuses`,
+     * so `new Set(["failed"])` makes a failed run throw. Empty by default.
+     */
     failureStatuses?: Set<string>;
     timeout?: number;
     pollInterval?: number;
@@ -541,6 +567,8 @@ type TaskInput = string | {
     url: string;
     external_id?: string;
     metadata?: Record<string, unknown>;
+    /** Per-task scrape params, merged over the job-level `zenrowsParams` (task wins). */
+    zenrows_params?: Record<string, unknown>;
 };
 /**
  * Client for the Zenrows Batch API (async, job/run/task model). Usable standalone
@@ -683,6 +711,187 @@ declare class ZenRowsBatchClient {
     }): Promise<Job>;
 }
 
+/** `running` until the crawl ends; the other three are terminal. */
+type CrawlStatus = "running" | "completed" | "stopped" | "failed";
+/** What ended a crawl early: a limit (`status: completed`) or `stop()` (`status: stopped`). */
+type CrawlStopReason = "max_items" | "max_pages" | "user";
+/** Why a crawl failed (`status: failed`). */
+type CrawlErrorCode = "insufficient_credits" | "seed_unreachable" | "domain_not_allowed" | "no_items_found" | "internal_error";
+/** Where a kept URL's page stands, when the crawl has an `output_format`. */
+type CrawlContentStatus = "pending" | "fetched" | "failed";
+/** Return each kept URL's page as fetched. Absent means URLs only. */
+type CrawlOutputFormat = "html";
+interface CrawlCoverage {
+    pages_fetched: number;
+    pages_failed: number;
+    items_found: number;
+}
+/** Why a crawl failed. Part of the crawl, not an error response. */
+interface CrawlRunError {
+    code: Extensible<CrawlErrorCode>;
+    detail: string;
+}
+interface Crawl {
+    crawl_id: string;
+    status: Extensible<CrawlStatus>;
+    url: string;
+    /** The depth limit the crawl was created with. Absent when it was created without one: no depth limit. */
+    depth?: number;
+    max_items: number;
+    max_pages: number;
+    coverage: CrawlCoverage;
+    created_at: string;
+    stop_reason?: Extensible<CrawlStopReason>;
+    error?: CrawlRunError;
+    include_patterns?: string[];
+    exclude_patterns?: string[];
+    output_format?: Extensible<CrawlOutputFormat>;
+    duplicates_removed?: number;
+    /** Absent while the crawl runs. */
+    finished_at?: string;
+}
+interface CrawlResult {
+    url: string;
+    content_status?: Extensible<CrawlContentStatus>;
+    /** The page's path, e.g. `/v1/crawls/c_x/contents/ct_y`. Present when `content_status` is `fetched`. */
+    content_url?: string;
+}
+interface CrawlWithResults extends Crawl {
+    results: CrawlResult[];
+    /** Never null while the crawl runs; null once it ended and this page holds its last URLs. */
+    next_cursor: string | null;
+}
+interface CrawlList {
+    crawls: Crawl[];
+    /** Absent on the last page. */
+    next_cursor?: string;
+}
+/** What `stop()` returns: where the crawl stands, without counts. */
+interface CrawlStop {
+    crawl_id: string;
+    status: Extensible<CrawlStatus>;
+    stop_reason?: Extensible<CrawlStopReason>;
+    error?: CrawlRunError;
+    finished_at?: string;
+}
+/** One line of a crawl's download. */
+interface CrawlExportLine {
+    url: string;
+    content_status?: Extensible<CrawlContentStatus>;
+    /** The page, present when `content_status` is `fetched`: HTML text, or an object for a JSON crawl. */
+    content?: string | Record<string, unknown>;
+}
+/** What `download()` returns. */
+interface CrawlDownload {
+    /**
+     * The crawl's status when the file was read (`X-Crawl-Status`; undefined without it).
+     * `running` means a later download may hold more lines.
+     */
+    status: Extensible<CrawlStatus> | undefined;
+    lines: AsyncIterable<CrawlExportLine>;
+}
+
+interface CrawlClientConfig {
+    /** Override the Crawl API base URL. Default `https://api.zenrows.com/v1`. */
+    baseURL?: string;
+    /**
+     * Retries for transient failures (502/503/504, network errors, 429 on reads) on idempotent
+     * requests, and for a `stop()` answered 429, 502, 503 or 504. Default 3.
+     */
+    retries?: number;
+    /** Seconds each HTTP request may take. Default 30. Does not limit reading a `download()` body. */
+    timeout?: number;
+}
+interface CreateCrawlParams {
+    /** The page the crawl starts from: an absolute public `http`/`https` URL. */
+    url: string;
+    /**
+     * Link hops to follow from the start URL, 1–100000. Omit it for no depth limit: the crawl
+     * follows links until `maxPages` or `maxItems` stops it, or no link is left to open.
+     */
+    depth?: number;
+    /** Stop once this many URLs are kept, 1–100000. Server default 100. */
+    maxItems?: number;
+    /** Stop once this many pages are fetched, 1–100000; bounds the cost. Server default 100. */
+    maxPages?: number;
+    /** Keep a URL only if it contains at least one of these substrings. */
+    includePatterns?: string[];
+    /** Drop a URL that contains any of these substrings, even if it matches an include pattern. */
+    excludePatterns?: string[];
+    /** Also fetch each kept URL's page. Absent means URLs only. */
+    outputFormat?: CrawlOutputFormat;
+}
+interface WaitForCrawlOptions {
+    /** Seconds to poll before returning the crawl as it stands. Default 600. */
+    timeout?: number;
+}
+/**
+ * Client for the Zenrows Crawl API (new): give it one start URL, read back the URLs behind it.
+ * Usable standalone (`new ZenRowsCrawlClient(apiKey)`) or via `client.crawl` on a `ZenRows`
+ * instance. A crawl runs asynchronously: `create()` returns at once, `wait()` polls until it
+ * ends, and `results()` / `download()` read what it kept.
+ * Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change.
+ */
+declare class ZenRowsCrawlClient {
+    readonly apiKey: string;
+    private readonly transport;
+    private readonly retries;
+    constructor(apiKey: string, config?: CrawlClientConfig);
+    /**
+     * Start a crawl. Sends only the fields you set. With an `idempotencyKey`, a retry returns the
+     * crawl the first request created instead of starting another, and 5xx and network failures
+     * are retried. A 429 `too_many_crawls` is never retried: it throws `ZenRowsCrawlError` with
+     * `retryAfter` set.
+     */
+    create(params: CreateCrawlParams, opts?: {
+        idempotencyKey?: string;
+    }): Promise<Crawl>;
+    /** Read a crawl's status, coverage and one page of results (`limit` 1–10000, default 1000). */
+    get(crawlId: string, options?: {
+        cursor?: string;
+        limit?: number;
+    }): Promise<CrawlWithResults>;
+    /** One page of the account's crawls, newest first (`limit` 1–100, default 20). */
+    list(options?: {
+        cursor?: string;
+        limit?: number;
+    }): Promise<CrawlList>;
+    /**
+     * Every URL the crawl kept, following `next_cursor` (`limit` is the page size). On a running
+     * crawl it yields the URLs kept so far and returns at the first empty page; it does not poll.
+     * Call `wait()` first to read every result.
+     */
+    results(crawlId: string, options?: {
+        limit?: number;
+    }): AsyncGenerator<CrawlResult>;
+    /**
+     * Stop a running crawl. Idempotent: a crawl that already ended answers as it ended.
+     * Because a stop is safe to repeat, `stop()` retries a 429, 502, 503 (including `crawl_busy`,
+     * which means the stop was not saved and the crawl is still running) or 504 up to `retries`
+     * times, waiting `Retry-After` seconds (at most 30; 1 when the header is missing or unusable)
+     * each time, then throws the last error.
+     */
+    stop(crawlId: string): Promise<CrawlStop>;
+    /**
+     * One kept URL's page as fetched (HTML). Pass the content id, a `content_url`, or the
+     * `CrawlResult` itself (its `content_url` is present once `content_status` is `fetched`).
+     * Throws `TypeError` when no content id can be read.
+     */
+    content(crawlId: string, contentId: string | CrawlResult): Promise<string>;
+    /**
+     * Every result of the crawl in one download: its `status` and the parsed lines (with the
+     * page when the crawl has an `output_format`). On a running crawl, `status` is `running` and
+     * the lines hold only what was kept so far.
+     */
+    download(crawlId: string): Promise<CrawlDownload>;
+    /**
+     * Poll until the crawl ends (any status but `running`) or `timeout` seconds pass, and return
+     * it, without results. A `failed` crawl resolves too: read `error`. On timeout the crawl is
+     * returned with status `running` and keeps running.
+     */
+    wait(crawlId: string, options?: WaitForCrawlOptions): Promise<Crawl>;
+}
+
 /**
  * A non-2xx response from the Batch API, decoded as RFC 7807 `application/problem+json`
  * where possible. `code` is the stable Problem `code` member (e.g. `file_input_not_found`) —
@@ -731,12 +940,35 @@ interface PollUntilOptions<T> {
  */
 declare function pollUntil<T>(fetch: () => Promise<T> | T, { isDone, isFailure, timeout, initialInterval, maxInterval, backoff, jitter, }: PollUntilOptions<T>): Promise<T>;
 
+/**
+ * The API's code for an account Crawl is not enabled on (403).
+ */
+declare const CRAWL_NOT_ENABLED_CODE = "REQS008";
+/**
+ * A non-2xx response from the Crawl API, decoded as RFC 9457 `application/problem+json`
+ * where possible. Branch on `status` and `code` (e.g. `crawl_not_found`, `too_many_crawls`,
+ * or `REQS008` when Crawl is not enabled for the account); `code` is undefined when the body
+ * has none.
+ */
+declare class ZenRowsCrawlError extends Error {
+    readonly status: number;
+    readonly problem: ProblemJson | undefined;
+    readonly code: string | undefined;
+    readonly detail: string | undefined;
+    readonly extras: Record<string, unknown> | undefined;
+    /** Seconds from the `Retry-After` header, e.g. 30 on a 429 `too_many_crawls`. */
+    readonly retryAfter: number | undefined;
+    constructor(status: number, problem: ProblemJson | undefined, extras?: Record<string, unknown>, retryAfter?: number);
+}
+
 type HttpMethods = "GET" | "POST" | "PUT";
 /** Extract mode — see https://docs.zenrows.com for what each mode returns. */
 type ExtractMode = "auto" | "native" | "standard";
 interface ClientConfig {
     concurrency?: number;
     retries?: number;
+    /** Config for `client.crawl`: its own `baseURL`, `retries` and `timeout`. */
+    crawl?: CrawlClientConfig;
 }
 interface ZenRowsConfig {
     /** Set to "auto" to enable Adaptive Stealth Mode — Zenrows starts with the cheapest
@@ -774,6 +1006,8 @@ declare class ZenRows {
     readonly fetchWithRetry: (input: string | URL | Request, init?: (RequestInit & fetch_retry.RequestInitRetryParams<typeof fetch>) | undefined) => Promise<Response>;
     /** Client for the Batch API (async job/run/task model) — see `./batch.ts`. */
     readonly batch: ZenRowsBatchClient;
+    /** Client for the Crawl API (new; one start URL in, the URLs behind it out). See `./crawl/client.ts`. */
+    readonly crawl: ZenRowsCrawlClient;
     constructor(apiKey: string, clientConfig?: ClientConfig);
     /**
      * Fetch a URL through Zenrows — the main page-scraping product. This is the primary entry
@@ -816,4 +1050,4 @@ declare class ZenRows {
     private worker;
 }
 
-export { AUTO_MAX_CREDITS, AUTO_MIN_CREDITS, type AddTasksResponse, At, BASE_CREDITS, type BatchClientConfig, type BatchTask, type BatchWebhookConfig, type Cadence, Calendar, type CostEstimate, type CostLine, type CreateJobInputResponse, CurrentRun, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_MAX_COUNT_IN_MEMORY, DEFAULT_MAX_FILES, DEFAULT_MAX_TOTAL_BYTES_IN_MEMORY, Daily, type DownloadToDirOptions, type DownloadToMemoryOptions, type DownloadedResult, type EstimateTask, type Export, ExportHandle, ExportRef, type ExportStatus, type ExtractMode, type FileInputUploadTarget, type GetResultsOptions, type HMACKeyCreated, type HMACKeyFinalized, type HMACKeyList, type HMACKeyMeta, type IngestStatus, JS_AND_PROXY_CREDITS, JS_CREDITS, type Job, JobHandle, JobRef, type JobSchedule, type JobStatus, type JobType, type ListJobRunsResponse, type ListJobsOptions, type ListJobsResponse, type ListResultsResponse, Monthly, PREMIUM_PROXY_CREDITS, type ParamMap, type ParamValue, type PauseState, type PollUntilOptions, type ProblemJson, Rate, type RerunJobResponse, type Run, RunHandle, RunRef, type RunStats, type RunStatus, type Schedule, type ScheduleState, type Spend, type StartExportResponse, type SubmitJobOptions, type SubmitJobResponse, type SubmitTypedOptions, type TaskHistoryEvent, type TaskHistoryResponse, type TaskLike, type TaskResult, type TaskSpend, type TaskStatus, type TestWebhookResponse, type Tier, type WaitForRunOptions, WaiterFailureError, WaiterTimeoutError, type WebhookConfig, Weekly, ZenRows, ZenRowsBatchClient, ZenRowsBatchError, type ZenRowsConfig, downloadTaskToFile, downloadTaskToMemory, downloadToDir, downloadToMemory, estimateCost, parseProblem, pollUntil };
+export { AUTO_MAX_CREDITS, AUTO_MIN_CREDITS, type AddTasksResponse, At, BASE_CREDITS, type BatchClientConfig, type BatchTask, type BatchWebhookConfig, CRAWL_NOT_ENABLED_CODE, type Cadence, Calendar, type CostEstimate, type CostLine, type Crawl, type CrawlClientConfig, type CrawlContentStatus, type CrawlCoverage, type CrawlDownload, type CrawlErrorCode, type CrawlExportLine, type CrawlList, type CrawlOutputFormat, type CrawlResult, type CrawlRunError, type CrawlStatus, type CrawlStop, type CrawlStopReason, type CrawlWithResults, type CreateCrawlParams, type CreateJobInputResponse, CurrentRun, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_MAX_COUNT_IN_MEMORY, DEFAULT_MAX_FILES, DEFAULT_MAX_TOTAL_BYTES_IN_MEMORY, Daily, type DownloadToDirOptions, type DownloadToMemoryOptions, type DownloadedResult, type EstimateTask, type Export, ExportHandle, ExportRef, type ExportStatus, type Extensible, type ExtractMode, type FileInputUploadTarget, type GetResultsOptions, type HMACKeyCreated, type HMACKeyFinalized, type HMACKeyList, type HMACKeyMeta, type IngestStatus, JS_AND_PROXY_CREDITS, JS_CREDITS, type Job, JobHandle, JobRef, type JobSchedule, type JobStatus, type JobType, type ListJobRunsResponse, type ListJobsOptions, type ListJobsResponse, type ListResultsResponse, Monthly, PREMIUM_PROXY_CREDITS, type ParamMap, type ParamValue, type PauseState, type PollUntilOptions, type ProblemJson, Rate, type RerunJobResponse, type Run, type RunFailureReason, RunHandle, RunRef, type RunStats, type RunStatus, type Schedule, type ScheduleState, type Spend, type StartExportResponse, type SubmitJobOptions, type SubmitJobResponse, type SubmitTypedOptions, type TaskHistoryEvent, type TaskHistoryResponse, type TaskLike, type TaskResult, type TaskSpend, type TaskStatus, type TestWebhookResponse, type Tier, type WaitForCrawlOptions, type WaitForRunOptions, WaiterFailureError, WaiterTimeoutError, type WebhookConfig, Weekly, ZenRows, ZenRowsBatchClient, ZenRowsBatchError, type ZenRowsConfig, ZenRowsCrawlClient, ZenRowsCrawlError, downloadTaskToFile, downloadTaskToMemory, downloadToDir, downloadToMemory, estimateCost, parseProblem, pollUntil };