webcrawlerapi-js 2.2.2 → 2.2.6

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, JobMarkdownResponse, AgentRunRequest, AgentRun } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, JobMarkdownResponse, MarkdownRequest, MarkdownResponse, AgentRunRequest, AgentRun } from "./model";
 import { JobStatus, ErrorCode } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus, ErrorCode };
@@ -20,5 +20,6 @@ export declare class WebcrawlerClient {
     getJob(jobID: string): Promise<Job>;
     getJobMarkdown(jobID: string): Promise<JobMarkdownResponse>;
     getJobMarkdownContent(jobID: string): Promise<string>;
+    markdown(request: MarkdownRequest): Promise<MarkdownResponse>;
     private sendRequest;
 }

package/dist/@types/model.d.ts

@@ -29,6 +29,8 @@ export interface ScrapeResponse {
     page_title?: string;
     structured_data?: any;
     links?: string[];
+    /** Server-recommended delay before the next status poll, if provided */
+    recommended_pull_delay_ms?: number;
 }
 export interface ScrapeResponseError {
     success: boolean;
@@ -120,6 +122,15 @@ export interface UploadS3Action extends Action {
 export interface JobMarkdownResponse {
     content_url: string;
 }
+export interface MarkdownRequest {
+    /** The URL of the webpage to extract the article markdown from. */
+    url: string;
+}
+export interface MarkdownResponse {
+    success: boolean;
+    /** Cleaned article markdown (main content only). */
+    markdown?: string;
+}
 export interface AgentRunRequest {
     /** Natural-language task for the agent to complete. */
     prompt: string;

package/dist/api.js

@@ -24,6 +24,9 @@ const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
 const DEFAULT_POLL_DELAY_SECONDS = 2;
+const SCRAPE_MAX_WAIT_MS = 400 * 1000;
+// Effectively unbounded by count at the 2s default; elapsed time is the real limit
+const SCRAPE_MAX_POLLS = 1000;
 const AgentTerminalStatuses = ["done", "error", "canceled"];
 __exportStar(require("./model"), exports);
 const SCRAPE_VERSION = "v2";
@@ -183,17 +186,21 @@ class WebcrawlerClient {
             return {
                 success: false,
                 status: status,
-                page_status_code: 0
+                page_status_code: 0,
+                recommended_pull_delay_ms: responseData.recommended_pull_delay_ms
             };
         }
     }
-    async scrape(request, maxPolls = 100) {
+    async scrape(request, maxPolls = SCRAPE_MAX_POLLS) {
         // Start the scraping job
         const scrapeIdResponse = await this.scrapeAsync(request);
         const scrapeId = scrapeIdResponse.id;
+        const startedAt = Date.now();
         let polls = 0;
+        let delayMs = DEFAULT_POLL_DELAY_SECONDS * 1000;
         let result;
-        while (polls < maxPolls) {
+        // Bounded by both elapsed time (SCRAPE_MAX_WAIT_MS) and maxPolls
+        while (polls < maxPolls && Date.now() - startedAt < SCRAPE_MAX_WAIT_MS) {
             result = await this.getScrape(scrapeId);
             // Return immediately if scrape is done
             if ('status' in result && result.status === "done") {
@@ -203,12 +210,17 @@ class WebcrawlerClient {
             if ('error_code' in result) {
                 return result;
             }
-            // Continue polling if status is in_progress or any other non-terminal status
-            // Wait before next poll
-            await new Promise(resolve => setTimeout(resolve, DEFAULT_POLL_DELAY_SECONDS * 1000));
+            // Use server-recommended delay when present, otherwise keep the default
+            const recommended = result.recommended_pull_delay_ms;
+            delayMs = recommended && recommended > 0 ? recommended : DEFAULT_POLL_DELAY_SECONDS * 1000;
+            // Don't sleep past the overall deadline
+            const remainingMs = SCRAPE_MAX_WAIT_MS - (Date.now() - startedAt);
+            if (remainingMs <= 0)
+                break;
+            await new Promise(resolve => setTimeout(resolve, Math.min(delayMs, remainingMs)));
             polls++;
         }
-        // Return the last known state if maxPolls is reached
+        // Return the last known state on timeout
         return result;
     }
     async crawl(crawlRequest, maxPolls = MaxPullRetries) {
@@ -328,6 +340,15 @@ class WebcrawlerClient {
         }
         return response.text();
     }
+    async markdown(request) {
+        const url = `${this.basePath}/markdown`;
+        const requestOptions = {
+            'method': 'POST',
+            'headers': this.getDefaultHeaders(),
+            'body': JSON.stringify(request),
+        };
+        return this.sendRequest(url, requestOptions);
+    }
     async sendRequest(url, requestOptions) {
         let response;
         try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "2.2.2",
+  "version": "2.2.6",
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",