webcrawlerapi-js 1.1.7 → 2.0.0

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, Action } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action } from "./model";
 import { JobStatus } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus };
@@ -8,10 +8,9 @@ export declare class WebcrawlerClient {
     protected apiKey: string;
     protected basePath: string;
     constructor(apiKey: string, basePath?: string, apiVersion?: string);
-    scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
-    scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
-    scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
-    getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
+    scrapeAsync(request: ScrapeRequest): Promise<ScrapeId>;
+    getScrape(scrapeId: string): Promise<ScrapeResponse | ScrapeResponseError>;
+    scrape(request: ScrapeRequest, maxPolls?: number): Promise<ScrapeResponse | ScrapeResponseError>;
     crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;
     crawlAsync(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;

package/dist/@types/model.d.ts

@@ -1,21 +1,32 @@
 export interface ScrapeRequest {
-    input: any;
-    crawler_id: string;
-    max_retries?: number;
-    debug?: boolean;
-    omit_cache?: boolean;
+    url: string;
+    output_format: string;
+    webhook_url?: string;
+    clean_selectors?: string;
+    actions?: Action[];
+    prompt?: string;
 }
-export interface JobId {
+export interface ScrapeId {
     id: string;
 }
 export interface ScrapeResponse {
-    id: string;
-    url: string;
-    status: any;
+    success: boolean;
+    status?: string;
+    markdown?: string;
+    cleaned_content?: string;
+    raw_content?: string;
     page_status_code: number;
-    created_at: string;
-    structured_data: any;
-    recommended_pull_delay_ms: number;
+    page_title?: string;
+    structured_data?: any;
+}
+export interface ScrapeResponseError {
+    success: boolean;
+    error_code: string;
+    error_message: string;
+    status?: string;
+}
+export interface JobId {
+    id: string;
 }
 export interface CrawlRequest {
     url?: string;

package/dist/api.js

@@ -22,8 +22,9 @@ Object.defineProperty(exports, "WebcrawlerApiError", { enumerable: true, get: fu
 const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
+const DEFAULT_POLL_DELAY_SECONDS = 2;
 __exportStar(require("./model"), exports);
-// Helper function to add getContent method to job items
+const SCRAPE_VERSION = "v2";
 function addGetContentMethod(job) {
     // Transform each job item to include getContent method
     job.job_items = job.job_items.map(item => ({
@@ -68,8 +69,9 @@ class WebcrawlerClient {
         this.basePath = basePath;
         this.apiVersion = apiVersion;
     }
-    async scrapeAsync(scrapeRequest) {
-        const url = `${this.basePath}/${this.apiVersion}/scrape`;
+    async scrapeAsync(request) {
+        const apiUrl = `${this.basePath}/${SCRAPE_VERSION}/scrape?async=true`;
+        console.log(request);
         const requestOptions = {
             'method': 'POST',
             'headers': {
@@ -77,46 +79,13 @@ class WebcrawlerClient {
                 'Authorization': `Bearer ${this.apiKey}`,
                 "User-Agent": "WebcrawlerAPI-NodeJS-Client"
             },
-            'body': JSON.stringify(scrapeRequest),
+            'body': JSON.stringify(request),
         };
-        return await this.sendRequest(url, requestOptions);
-    }
-    async scrapeWithMeta(scrapeRequest, maxPollingRetries = MaxPullRetries) {
-        const url = `${this.basePath}/${this.apiVersion}/scrape`;
-        const requestOptions = {
-            'method': 'POST',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`
-            },
-            'body': JSON.stringify(scrapeRequest),
-        };
-        const jobIdResponse = await this.sendRequest(url, requestOptions);
-        if (jobIdResponse.id === '') {
-            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
-        }
-        let delayIntervalMs = initialPullDelayMs;
-        for (let i = 0; i < maxPollingRetries; i++) {
-            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-            const scrapeResult = await this.getScrapeResult(jobIdResponse.id);
-            if (scrapeRequest.debug) {
-                console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
-            }
-            if (scrapeResult.status !== constants_1.JobStatus.IN_PROGRESS && scrapeResult.status !== constants_1.JobStatus.NEW) {
-                return scrapeResult;
-            }
-            if (scrapeResult.recommended_pull_delay_ms > 0) {
-                delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
-            }
-        }
-        throw new errors_1.WebcrawlerApiError('timeout', 'Scraping took too long, please retry or increase the number of polling retries', 0);
-    }
-    async scrape(scrapeRequest, maxPollingRetries = MaxPullRetries) {
-        const scrapeResult = await this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
-        return scrapeResult.structured_data;
+        const response = await this.sendRequest(apiUrl, requestOptions);
+        return { id: response.id };
     }
-    async getScrapeResult(scrapeID) {
-        const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
+    async getScrape(scrapeId) {
+        const url = `${this.basePath}/${SCRAPE_VERSION}/scrape/${scrapeId}`;
         const requestOptions = {
             'method': 'GET',
             'headers': {
@@ -128,7 +97,46 @@ class WebcrawlerClient {
                 'Expires': '0'
             },
         };
-        return await this.sendRequest(url, requestOptions);
+        const responseData = await this.sendRequest(url, requestOptions);
+        const status = responseData.status;
+        if (status === "done") {
+            return responseData;
+        }
+        else if (status === "error") {
+            return responseData;
+        }
+        else {
+            // in_progress or any other status
+            return {
+                success: false,
+                status: status,
+                page_status_code: 0
+            };
+        }
+    }
+    async scrape(request, maxPolls = 100) {
+        // Start the scraping job
+        const scrapeIdResponse = await this.scrapeAsync(request);
+        const scrapeId = scrapeIdResponse.id;
+        let polls = 0;
+        let result;
+        while (polls < maxPolls) {
+            result = await this.getScrape(scrapeId);
+            // Return immediately if scrape is done
+            if ('status' in result && result.status === "done") {
+                return result;
+            }
+            // Return immediately if there's an error
+            if ('error_code' in result) {
+                return result;
+            }
+            // Continue polling if status is in_progress or any other non-terminal status
+            // Wait before next poll
+            await new Promise(resolve => setTimeout(resolve, DEFAULT_POLL_DELAY_SECONDS * 1000));
+            polls++;
+        }
+        // Return the last known state if maxPolls is reached
+        return result;
     }
     async crawl(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.1.7",
+  "version": "2.0.0",
 
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",