webcrawlerapi-js 2.2.5 → 2.2.7

package/README.md

@@ -46,6 +46,23 @@ for (const item of job.job_items) {
 }
 ```
 
+## Batch scrape
+
+```javascript
+const batch = await client.scrape({
+    url: ["https://books.toscrape.com/", "https://books.toscrape.com/catalogue/page-2.html"],
+    output_formats: ["markdown"],
+    max_concurrency: 5,
+})
+
+console.log(batch.status, batch.completed, batch.failed)
+for (const item of batch.data) console.log(item.url, item.status, item.markdown_url)
+```
+
+Passing an array as `url` to `scrape()` creates a batch and polls until it finishes (a single string still returns a single result). Batch-only options like `max_concurrency` and `ignore_invalid_urls` go in the same request. For manual control use `scrapeBatchAsync({ urls, ... })`, `getBatch(id, { limit, cursor })`, `getBatchErrors(id)` and `cancelBatch(id)`.
+
+Full batch scrape documentation: [docs/batch-scrape.md](docs/batch-scrape.md).
+
 ## Run an agent
 
 ```javascript

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, JobMarkdownResponse, MarkdownRequest, MarkdownResponse, AgentRunRequest, AgentRun } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, JobMarkdownResponse, MarkdownRequest, MarkdownResponse, AgentRunRequest, AgentRun, BatchScrapeRequest, BatchScrapeId, BatchScrape, BatchScrapeGetOptions, BatchScrapeErrors, ScrapeBatchRequest } from "./model";
 import { JobStatus, ErrorCode } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus, ErrorCode };
@@ -14,7 +14,26 @@ export declare class WebcrawlerClient {
     getAgentJob(jobID: string): Promise<AgentRun>;
     runAgent(request: AgentRunRequest, maxPolls?: number): Promise<AgentRun>;
     getScrape(scrapeId: string): Promise<ScrapeResponse | ScrapeResponseError>;
+    /**
+     * Scrapes a single URL and waits for the result.
+     *
+     * Passing `url` as an array creates a batch scrape instead, polls until it reaches a
+     * terminal status (completed, failed or cancelled) and returns the first page of items;
+     * follow `next` with getBatch(id, { cursor }) for larger batches. Batch-only options
+     * (max_concurrency, ignore_invalid_urls, ...) go in the same request object. A batch throws a
+     * `timeout` error if it does not finish within an hour or maxPolls.
+     * For manual batch control use scrapeBatchAsync() + getBatch().
+     */
     scrape(request: ScrapeRequest, maxPolls?: number): Promise<ScrapeResponse | ScrapeResponseError>;
+    scrape(request: ScrapeBatchRequest, maxPolls?: number): Promise<BatchScrape>;
+    private scrapeSingle;
+    scrapeBatchAsync(request: BatchScrapeRequest): Promise<BatchScrapeId>;
+    getBatch(batchId: string, options?: BatchScrapeGetOptions): Promise<BatchScrape>;
+    getBatchErrors(batchId: string): Promise<BatchScrapeErrors>;
+    cancelBatch(batchId: string): Promise<{
+        status: string;
+    }>;
+    private scrapeBatchAndWait;
     crawl(crawlRequest: CrawlRequest, maxPolls?: number): Promise<Job>;
     crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;

package/dist/@types/model.d.ts

@@ -29,6 +29,8 @@ export interface ScrapeResponse {
     page_title?: string;
     structured_data?: any;
     links?: string[];
+    /** Server-recommended delay before the next status poll, if provided */
+    recommended_pull_delay_ms?: number;
 }
 export interface ScrapeResponseError {
     success: boolean;
@@ -161,3 +163,76 @@ export interface AgentRunList {
     items: AgentRun[];
     total: number;
 }
+export type BatchScrapeStatus = 'scraping' | 'completed' | 'failed' | 'cancelled';
+export interface BatchScrapeRequest {
+    /** URLs to scrape (max is your plan's job items limit). */
+    urls: string[];
+    /** Default is ["markdown"]. */
+    output_formats?: ('markdown' | 'cleaned' | 'html' | 'links' | 'images')[];
+    main_content_only?: boolean;
+    /** CSS selectors to clean from the output. */
+    clean_selectors?: string;
+    /** Max URLs scraped in parallel for this batch. 0 or omitted means no per-batch cap. */
+    max_concurrency?: number;
+    /** Skip invalid URLs and report them in invalid_urls instead of failing the request (default: true). */
+    ignore_invalid_urls?: boolean;
+    /** Public http(s) URL called once when the batch finishes. */
+    webhook_url?: string;
+    /** Cache duration in seconds (default 604800). 0 bypasses the cache. */
+    max_age?: number;
+    respect_robots_txt?: boolean;
+}
+/** Request for scrape() with an array `url`: runs a batch scrape and waits for it. Same options as BatchScrapeRequest. */
+export type ScrapeBatchRequest = Omit<BatchScrapeRequest, 'urls'> & {
+    /** URLs to scrape as a batch (max is your plan's job items limit). */
+    url: string[];
+};
+export interface BatchScrapeInvalidUrl {
+    url: string;
+    error_code: string;
+    error_message: string;
+}
+export interface BatchScrapeId {
+    id: string;
+    status_url: string;
+    invalid_urls: BatchScrapeInvalidUrl[];
+}
+export interface BatchScrapeItem {
+    url: string;
+    /** Item status, for example "done" or "error". */
+    status: string;
+    page_status_code?: number;
+    title?: string;
+    markdown_url?: string;
+    cleaned_url?: string;
+    html_url?: string;
+    from_cache?: boolean;
+    error_code?: string;
+    error_message?: string;
+}
+export interface BatchScrape {
+    id: string;
+    status: BatchScrapeStatus | string;
+    total: number;
+    completed: number;
+    failed: number;
+    canceled: number;
+    data: BatchScrapeItem[];
+    /** Cursor for the next page. Empty on the last page. */
+    next?: string;
+}
+export interface BatchScrapeGetOptions {
+    /** Items per page, 1 to 500 (default 100). */
+    limit?: number;
+    /** Value of `next` from the previous page. */
+    cursor?: string;
+}
+export interface BatchScrapeError {
+    url: string;
+    error_code: string;
+    error_message: string;
+}
+export interface BatchScrapeErrors {
+    errors: BatchScrapeError[];
+    robots_blocked: string[];
+}

package/dist/api.js

@@ -24,6 +24,12 @@ const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
 const DEFAULT_POLL_DELAY_SECONDS = 2;
+const SCRAPE_MAX_WAIT_MS = 400 * 1000;
+// Effectively unbounded by count at the 2s default; elapsed time is the real limit
+const SCRAPE_MAX_POLLS = 1000;
+const BATCH_VERSION = "v2";
+const BATCH_MAX_WAIT_MS = 3600 * 1000;
+const BatchTerminalStatuses = ["completed", "failed", "cancelled", "canceled"];
 const AgentTerminalStatuses = ["done", "error", "canceled"];
 __exportStar(require("./model"), exports);
 const SCRAPE_VERSION = "v2";
@@ -183,17 +189,28 @@ class WebcrawlerClient {
             return {
                 success: false,
                 status: status,
-                page_status_code: 0
+                page_status_code: 0,
+                recommended_pull_delay_ms: responseData.recommended_pull_delay_ms
             };
         }
     }
-    async scrape(request, maxPolls = 100) {
+    async scrape(request, maxPolls = SCRAPE_MAX_POLLS) {
+        if (Array.isArray(request.url)) {
+            const { url, ...options } = request;
+            return await this.scrapeBatchAndWait({ ...options, urls: url }, maxPolls);
+        }
+        return await this.scrapeSingle(request, maxPolls);
+    }
+    async scrapeSingle(request, maxPolls) {
         // Start the scraping job
         const scrapeIdResponse = await this.scrapeAsync(request);
         const scrapeId = scrapeIdResponse.id;
+        const startedAt = Date.now();
         let polls = 0;
+        let delayMs = DEFAULT_POLL_DELAY_SECONDS * 1000;
         let result;
-        while (polls < maxPolls) {
+        // Bounded by both elapsed time (SCRAPE_MAX_WAIT_MS) and maxPolls
+        while (polls < maxPolls && Date.now() - startedAt < SCRAPE_MAX_WAIT_MS) {
             result = await this.getScrape(scrapeId);
             // Return immediately if scrape is done
             if ('status' in result && result.status === "done") {
@@ -203,14 +220,73 @@ class WebcrawlerClient {
             if ('error_code' in result) {
                 return result;
             }
-            // Continue polling if status is in_progress or any other non-terminal status
-            // Wait before next poll
-            await new Promise(resolve => setTimeout(resolve, DEFAULT_POLL_DELAY_SECONDS * 1000));
+            // Use server-recommended delay when present, otherwise keep the default
+            const recommended = result.recommended_pull_delay_ms;
+            delayMs = recommended && recommended > 0 ? recommended : DEFAULT_POLL_DELAY_SECONDS * 1000;
+            // Don't sleep past the overall deadline
+            const remainingMs = SCRAPE_MAX_WAIT_MS - (Date.now() - startedAt);
+            if (remainingMs <= 0)
+                break;
+            await new Promise(resolve => setTimeout(resolve, Math.min(delayMs, remainingMs)));
             polls++;
         }
-        // Return the last known state if maxPolls is reached
+        // Return the last known state on timeout
         return result;
     }
+    async scrapeBatchAsync(request) {
+        const url = `${this.basePath}/${BATCH_VERSION}/batch/scrape`;
+        const requestOptions = {
+            'method': 'POST',
+            'headers': this.getDefaultHeaders(),
+            'body': JSON.stringify(request),
+        };
+        return await this.sendRequest(url, requestOptions);
+    }
+    async getBatch(batchId, options = {}) {
+        const params = new URLSearchParams();
+        if (options.limit !== undefined)
+            params.set('limit', String(options.limit));
+        if (options.cursor)
+            params.set('cursor', options.cursor);
+        const query = params.toString();
+        const url = `${this.basePath}/${BATCH_VERSION}/batch/scrape/${batchId}${query ? `?${query}` : ''}`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': this.getDefaultHeaders(true),
+        };
+        return await this.sendRequest(url, requestOptions);
+    }
+    async getBatchErrors(batchId) {
+        const url = `${this.basePath}/${BATCH_VERSION}/batch/scrape/${batchId}/errors`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': this.getDefaultHeaders(true),
+        };
+        return await this.sendRequest(url, requestOptions);
+    }
+    async cancelBatch(batchId) {
+        const url = `${this.basePath}/${BATCH_VERSION}/batch/scrape/${batchId}`;
+        const requestOptions = {
+            'method': 'DELETE',
+            'headers': this.getDefaultHeaders(),
+        };
+        return await this.sendRequest(url, requestOptions);
+    }
+    async scrapeBatchAndWait(request, maxPolls) {
+        const created = await this.scrapeBatchAsync(request);
+        if (!created.id) {
+            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to create batch scrape', 0);
+        }
+        const startedAt = Date.now();
+        for (let i = 0; i < maxPolls && Date.now() - startedAt < BATCH_MAX_WAIT_MS; i++) {
+            const batch = await this.getBatch(created.id);
+            if (BatchTerminalStatuses.includes(batch.status)) {
+                return batch;
+            }
+            await new Promise(resolve => setTimeout(resolve, DEFAULT_POLL_DELAY_SECONDS * 1000));
+        }
+        throw new errors_1.WebcrawlerApiError('timeout', 'Batch scrape took too long, use scrapeBatchAsync + getBatch to keep polling', 0);
+    }
     async crawl(crawlRequest, maxPolls = MaxPullRetries) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
         const requestBody = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "2.2.5",
+  "version": "2.2.7",
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",