webcrawlerapi-js 2.2.5 → 2.2.6

package/dist/@types/model.d.ts

@@ -29,6 +29,8 @@ export interface ScrapeResponse {
     page_title?: string;
     structured_data?: any;
     links?: string[];
+    /** Server-recommended delay before the next status poll, if provided */
+    recommended_pull_delay_ms?: number;
 }
 export interface ScrapeResponseError {
     success: boolean;

package/dist/api.js

@@ -24,6 +24,9 @@ const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
 const DEFAULT_POLL_DELAY_SECONDS = 2;
+const SCRAPE_MAX_WAIT_MS = 400 * 1000;
+// Effectively unbounded by count at the 2s default; elapsed time is the real limit
+const SCRAPE_MAX_POLLS = 1000;
 const AgentTerminalStatuses = ["done", "error", "canceled"];
 __exportStar(require("./model"), exports);
 const SCRAPE_VERSION = "v2";
@@ -183,17 +186,21 @@ class WebcrawlerClient {
             return {
                 success: false,
                 status: status,
-                page_status_code: 0
+                page_status_code: 0,
+                recommended_pull_delay_ms: responseData.recommended_pull_delay_ms
             };
         }
     }
-    async scrape(request, maxPolls = 100) {
+    async scrape(request, maxPolls = SCRAPE_MAX_POLLS) {
         // Start the scraping job
         const scrapeIdResponse = await this.scrapeAsync(request);
         const scrapeId = scrapeIdResponse.id;
+        const startedAt = Date.now();
         let polls = 0;
+        let delayMs = DEFAULT_POLL_DELAY_SECONDS * 1000;
         let result;
-        while (polls < maxPolls) {
+        // Bounded by both elapsed time (SCRAPE_MAX_WAIT_MS) and maxPolls
+        while (polls < maxPolls && Date.now() - startedAt < SCRAPE_MAX_WAIT_MS) {
             result = await this.getScrape(scrapeId);
             // Return immediately if scrape is done
             if ('status' in result && result.status === "done") {
@@ -203,12 +210,17 @@ class WebcrawlerClient {
             if ('error_code' in result) {
                 return result;
             }
-            // Continue polling if status is in_progress or any other non-terminal status
-            // Wait before next poll
-            await new Promise(resolve => setTimeout(resolve, DEFAULT_POLL_DELAY_SECONDS * 1000));
+            // Use server-recommended delay when present, otherwise keep the default
+            const recommended = result.recommended_pull_delay_ms;
+            delayMs = recommended && recommended > 0 ? recommended : DEFAULT_POLL_DELAY_SECONDS * 1000;
+            // Don't sleep past the overall deadline
+            const remainingMs = SCRAPE_MAX_WAIT_MS - (Date.now() - startedAt);
+            if (remainingMs <= 0)
+                break;
+            await new Promise(resolve => setTimeout(resolve, Math.min(delayMs, remainingMs)));
             polls++;
         }
-        // Return the last known state if maxPolls is reached
+        // Return the last known state on timeout
         return result;
     }
     async crawl(crawlRequest, maxPolls = MaxPullRetries) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "2.2.5",
+  "version": "2.2.6",
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",