webcrawlerapi-js 1.1.6 → 2.0.0

package/README.md

@@ -20,34 +20,36 @@ The package is regularly updated with new features and improvements to ensure th
 ## JavaScript Examples
 
 ### Synchronous Job
-In this mode, the promise will be resolved when all data is ready. Synchronous jobs are perfect for quick scraping tasks where you need immediate results and don't want to handle polling logic.
+In this mode, the promise will be resolved when all data is ready. Synchronous jobs are perfect for quick scraping tasks where you need immediate results and don't want to handle polling logic. However, for long running jobs we recommend to use async crawling or the webhook.
 
 ```javascript
 import webcrawlerapi from "webcrawlerapi-js"
 
 async function main() {
-    const client = new WebcrawlerClient(
-        "YOUR API ACCESS KEY HERE",
-    )
+    const client = new WebcrawlerClient("YOUR API ACCESS KEY HERE")
 
     try {
-        const syncJob = await client.crawl({
+        const crawlJob = await client.crawl({
             "items_limit": 10,
-            "url": "https://stripe.com/",
+            "url": "https://books.toscrape.com/",
             "scrape_type": "markdown"
         })
         
-        for (const item of syncJob.job_items) {
+        // Get job details
+        const job = await client.getJob(crawlJob.id);
+        console.log("Job status:", job.status);
+
+        // Output item content
+        for (const item of job.job_items) {
             const content = await item.getContent()
+            console.log("---- Title: " + item.title + "----")
             console.log(content.slice(0, 100))
         }
-        console.log(syncJob)
-        
     } catch (error) {
         if (error instanceof webcrawlerapi.WebcrawlerApiError) {
             console.error("Webcrawler error:", error.message)
         }
-        return
+        console.error(error)
     }
 }
 
@@ -61,27 +63,32 @@ In this mode, you get a job ID immediately and can poll for results. Asynchronou
 import webcrawlerapi from "webcrawlerapi-js"
 
 async function main() {
-    const client = new WebcrawlerClient(
-        "YOUR API ACCESS KEY HERE",
-    )
+    const client = new WebcrawlerClient("YOUR API ACCESS KEY HERE")
 
     try {
-        // Get job ID immediately
+        try {
         const jobWithId = await client.crawlAsync({
             "items_limit": 10,
-            "url": "https://stripe.com/",
+            "url": "https://books.toscrape.com/",
             "scrape_type": "markdown"
         })
-        
+
         // Poll for job completion
         const jobId = jobWithId.id
         let asyncJob = await client.getJob(jobId)
-        
+
         // Check job status
-        console.log("Job status:", asyncJob.status)
-        
+        console.log("Job id:", asyncJob.id)
+        console.log("Dashboard link:", `https://dash.webcrawlerapi.com/jobs/job/${asyncJob.id}`)
+
+        //wait until job is done
+        while (asyncJob.status !== "done") {
+            await new Promise(resolve => setTimeout(resolve, 1000));
+            asyncJob = await client.getJob(jobId)
+        }
+
         // Once complete, access results
-        if (asyncJob.status === "completed") {
+        if (asyncJob.status === "done") {
             for (const item of asyncJob.job_items) {
                 const content = await item.getContent()
                 console.log(content.slice(0, 100))
@@ -91,6 +98,8 @@ async function main() {
         if (error instanceof webcrawlerapi.WebcrawlerApiError) {
             console.error("Webcrawler error:", error.message)
         }
+        console.error(error)
+        return
     }
 }
 
@@ -102,87 +111,40 @@ main().catch(console.error)
 The SDK provides full TypeScript support with type safety and error handling. Our TypeScript integration ensures you catch potential issues at compile-time and provides excellent IDE support with autocompletion and inline documentation.
 
 ```typescript
-import { WebcrawlerClient, WebcrawlerApiError } from "webcrawlerapi-js"
+import {WebcrawlerClient, WebcrawlerApiError} from "webcrawlerapi-js";
 
 async function main() {
-    const client = new WebcrawlerClient(
-        "YOUR_API_KEY",
-        // Optional base URL, defaults to production API
-        "http://localhost:8080" 
-    )
+    const client = new WebcrawlerClient("YOUR_API_KEY")
 
     try {
         // Basic crawl with markdown
         const syncJob = await client.crawl({
-            "items_limit": 2,
+            "items_limit": 10,
             "url": "https://books.toscrape.com/",
-            "scrape_type": "markdown"
-        })
-
-        // Type-safe status check
-        if (syncJob.status === "done") {
-            for (const item of syncJob.job_items) {
-                if (item.markdown_content_url) {
-                    // Access typed properties
-                    console.log("Status code:", item.page_status_code)
-                    console.log("Content URL:", item.markdown_content_url)
-                }
-            }
+            "scrape_type": "markdown",
         }
-
-        // Using specialized scraper
-        const scrapeJob = await client.scrape({
-            "crawler_id": "webcrawler/url-to-md",
-            "input": {
-                "url": "https://books.toscrape.com/"
-            }
-        })
-
-        if (scrapeJob.status === "done") {
-            const content = scrapeJob.getContent()
-            console.log("Scraped content:", content)
+    )
+        // Type-safe status check
+        for (const item of syncJob.job_items) {
+            const content = await item.getContent()
+            console.log(content.slice(0,100))
         }
-
     } catch (error) {
-        // Type-safe error handling
         if (error instanceof WebcrawlerApiError) {
             console.error("API Error:", error.errorCode, error.message)
-            // errorCode is typed and can be used for specific error handling
-            if (error.errorCode === "invalid_request") {
-                // Handle invalid request
-            }
         }
+        console.log(error)
     }
 }
 
 main().catch(console.error)
 ```
 
-### Error Handling
-
-The SDK provides typed error handling through the `WebcrawlerApiError` class. Our error handling system is designed to provide detailed, actionable feedback that helps you quickly identify and resolve issues in your scraping pipeline.
-
-```typescript
-try {
-    const syncJob = await client.crawl({
-        "items_limit": 2,
-        "url": "https://books.toscrape.com/",
-        // This will cause an error due to missing scrape_type
-    })
-} catch (error) {
-    if (error instanceof WebcrawlerApiError) {
-        // Typed error properties
-        console.error("Error code:", error.errorCode)
-        console.error("Error message:", error.message)
-    }
-}
-```
-
 ## Response Structure
 
 Example of a job response. The response structure is carefully designed to provide comprehensive information about your scraping job, including detailed status updates, timing information, and resource usage metrics.
 
-```json
+```js
 {
   id: '49c4942b-b7d9-4d62-94b5-b54a3016ac51',
   url: 'https://stripe.com/',

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, Action } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action } from "./model";
 import { JobStatus } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus };
@@ -8,10 +8,9 @@ export declare class WebcrawlerClient {
     protected apiKey: string;
     protected basePath: string;
     constructor(apiKey: string, basePath?: string, apiVersion?: string);
-    scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
-    scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
-    scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
-    getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
+    scrapeAsync(request: ScrapeRequest): Promise<ScrapeId>;
+    getScrape(scrapeId: string): Promise<ScrapeResponse | ScrapeResponseError>;
+    scrape(request: ScrapeRequest, maxPolls?: number): Promise<ScrapeResponse | ScrapeResponseError>;
     crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;
     crawlAsync(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;

package/dist/@types/model.d.ts

@@ -1,21 +1,32 @@
 export interface ScrapeRequest {
-    input: any;
-    crawler_id: string;
-    max_retries?: number;
-    debug?: boolean;
-    omit_cache?: boolean;
+    url: string;
+    output_format: string;
+    webhook_url?: string;
+    clean_selectors?: string;
+    actions?: Action[];
+    prompt?: string;
 }
-export interface JobId {
+export interface ScrapeId {
     id: string;
 }
 export interface ScrapeResponse {
-    id: string;
-    url: string;
-    status: any;
+    success: boolean;
+    status?: string;
+    markdown?: string;
+    cleaned_content?: string;
+    raw_content?: string;
     page_status_code: number;
-    created_at: string;
-    structured_data: any;
-    recommended_pull_delay_ms: number;
+    page_title?: string;
+    structured_data?: any;
+}
+export interface ScrapeResponseError {
+    success: boolean;
+    error_code: string;
+    error_message: string;
+    status?: string;
+}
+export interface JobId {
+    id: string;
 }
 export interface CrawlRequest {
     url?: string;

package/dist/api.js

@@ -22,7 +22,46 @@ Object.defineProperty(exports, "WebcrawlerApiError", { enumerable: true, get: fu
 const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
+const DEFAULT_POLL_DELAY_SECONDS = 2;
 __exportStar(require("./model"), exports);
+const SCRAPE_VERSION = "v2";
+function addGetContentMethod(job) {
+    // Transform each job item to include getContent method
+    job.job_items = job.job_items.map(item => ({
+        ...item,
+        getContent: async function () {
+            if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
+                return null;
+            }
+            let contentUrl;
+            switch (job.scrape_type) {
+                case 'html':
+                    contentUrl = this.raw_content_url;
+                    break;
+                case 'cleaned':
+                    contentUrl = this.cleaned_content_url;
+                    break;
+                case 'markdown':
+                    contentUrl = this.markdown_content_url;
+                    break;
+            }
+            if (!contentUrl) {
+                return null;
+            }
+            const response = await fetch(contentUrl, {
+                headers: {
+                    'Accept-Encoding': 'gzip, deflate, br',
+                    'Accept': '*/*'
+                }
+            });
+            if (!response.ok) {
+                throw new Error(`Failed to fetch content: ${response.statusText}`);
+            }
+            return await response.text();
+        }
+    }));
+    return job;
+}
 class WebcrawlerClient {
     constructor(apiKey, basePath = BASE_PATH, apiVersion = "v1") {
         this.apiVersion = "v1";
@@ -30,8 +69,9 @@ class WebcrawlerClient {
         this.basePath = basePath;
         this.apiVersion = apiVersion;
     }
-    async scrapeAsync(scrapeRequest) {
-        const url = `${this.basePath}/${this.apiVersion}/scrape`;
+    async scrapeAsync(request) {
+        const apiUrl = `${this.basePath}/${SCRAPE_VERSION}/scrape?async=true`;
+        console.log(request);
         const requestOptions = {
             'method': 'POST',
             'headers': {
@@ -39,46 +79,13 @@ class WebcrawlerClient {
                 'Authorization': `Bearer ${this.apiKey}`,
                 "User-Agent": "WebcrawlerAPI-NodeJS-Client"
             },
-            'body': JSON.stringify(scrapeRequest),
-        };
-        return await this.sendRequest(url, requestOptions);
-    }
-    async scrapeWithMeta(scrapeRequest, maxPollingRetries = MaxPullRetries) {
-        const url = `${this.basePath}/${this.apiVersion}/scrape`;
-        const requestOptions = {
-            'method': 'POST',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`
-            },
-            'body': JSON.stringify(scrapeRequest),
+            'body': JSON.stringify(request),
         };
-        const jobIdResponse = await this.sendRequest(url, requestOptions);
-        if (jobIdResponse.id === '') {
-            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
-        }
-        let delayIntervalMs = initialPullDelayMs;
-        for (let i = 0; i < maxPollingRetries; i++) {
-            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-            const scrapeResult = await this.getScrapeResult(jobIdResponse.id);
-            if (scrapeRequest.debug) {
-                console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
-            }
-            if (scrapeResult.status !== constants_1.JobStatus.IN_PROGRESS && scrapeResult.status !== constants_1.JobStatus.NEW) {
-                return scrapeResult;
-            }
-            if (scrapeResult.recommended_pull_delay_ms > 0) {
-                delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
-            }
-        }
-        throw new errors_1.WebcrawlerApiError('timeout', 'Scraping took too long, please retry or increase the number of polling retries', 0);
-    }
-    async scrape(scrapeRequest, maxPollingRetries = MaxPullRetries) {
-        const scrapeResult = await this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
-        return scrapeResult.structured_data;
+        const response = await this.sendRequest(apiUrl, requestOptions);
+        return { id: response.id };
     }
-    async getScrapeResult(scrapeID) {
-        const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
+    async getScrape(scrapeId) {
+        const url = `${this.basePath}/${SCRAPE_VERSION}/scrape/${scrapeId}`;
         const requestOptions = {
             'method': 'GET',
             'headers': {
@@ -90,7 +97,46 @@ class WebcrawlerClient {
                 'Expires': '0'
             },
         };
-        return await this.sendRequest(url, requestOptions);
+        const responseData = await this.sendRequest(url, requestOptions);
+        const status = responseData.status;
+        if (status === "done") {
+            return responseData;
+        }
+        else if (status === "error") {
+            return responseData;
+        }
+        else {
+            // in_progress or any other status
+            return {
+                success: false,
+                status: status,
+                page_status_code: 0
+            };
+        }
+    }
+    async scrape(request, maxPolls = 100) {
+        // Start the scraping job
+        const scrapeIdResponse = await this.scrapeAsync(request);
+        const scrapeId = scrapeIdResponse.id;
+        let polls = 0;
+        let result;
+        while (polls < maxPolls) {
+            result = await this.getScrape(scrapeId);
+            // Return immediately if scrape is done
+            if ('status' in result && result.status === "done") {
+                return result;
+            }
+            // Return immediately if there's an error
+            if ('error_code' in result) {
+                return result;
+            }
+            // Continue polling if status is in_progress or any other non-terminal status
+            // Wait before next poll
+            await new Promise(resolve => setTimeout(resolve, DEFAULT_POLL_DELAY_SECONDS * 1000));
+            polls++;
+        }
+        // Return the last known state if maxPolls is reached
+        return result;
     }
     async crawl(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
@@ -120,40 +166,6 @@ class WebcrawlerClient {
             const timestamp = new Date().getTime();
             const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
             if (job.status !== constants_1.JobStatus.IN_PROGRESS && job.status !== constants_1.JobStatus.NEW) {
-                // Transform each job item to include getContent method
-                job.job_items = job.job_items.map(item => ({
-                    ...item,
-                    getContent: async function () {
-                        if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
-                            return null;
-                        }
-                        let contentUrl;
-                        switch (job.scrape_type) {
-                            case 'html':
-                                contentUrl = this.raw_content_url;
-                                break;
-                            case 'cleaned':
-                                contentUrl = this.cleaned_content_url;
-                                break;
-                            case 'markdown':
-                                contentUrl = this.markdown_content_url;
-                                break;
-                        }
-                        if (!contentUrl) {
-                            return null;
-                        }
-                        const response = await fetch(contentUrl, {
-                            headers: {
-                                'Accept-Encoding': 'gzip, deflate, br',
-                                'Accept': '*/*'
-                            }
-                        });
-                        if (!response.ok) {
-                            throw new Error(`Failed to fetch content: ${response.statusText}`);
-                        }
-                        return await response.text();
-                    }
-                }));
                 return job;
             }
             if (job.recommended_pull_delay_ms > 0) {
@@ -191,7 +203,8 @@ class WebcrawlerClient {
                 'Expires': '0'
             }
         };
-        return await this.sendRequest(url, requestOptions);
+        const job = await this.sendRequest(url, requestOptions);
+        return addGetContentMethod(job);
     }
     async sendRequest(url, requestOptions) {
         let response;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.1.6",
+  "version": "2.0.0",
 
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",