webcrawlerapi-js 1.0.3 → 1.0.5

package/README.md

@@ -1,60 +1,85 @@
-# JS client for WebcrawlerAPI scrapers
+# JS client for WebcrawlerAPI
 
-Official client for [WebcrawlerAPI](https://webcrawlerapi.com/) scrapers.
+Official client for [WebcrawlerAPI](https://webcrawlerapi.com/).
 
-## Installation
-Install WebcrawlerAPI js package:
-
-`npm i webcrawlerapi-js`
+WebcrawlerAPI allows you to extract data from any website with just a simple API call.
 
 ## Preparation
 1. Register to [dashboard|https://dash.webcrawlerapi.com/].
 2. Get an [Access Key](https://dash.webcrawlerapi.com/access).
 
+## Installation
+Install WebcrawlerAPI js package:
+
+`npm i webcrawlerapi-js`
+
 ## Request example
 
 ```javascript
-const webcrawlerapi = require('webcrawlerapi-js');
+import webcrawlerapi from "webcrawlerapi-js";
 
 async function main() {
     const client = new webcrawlerapi.WebcrawlerClient(
-        "YOUR API KEY HERE"
+        "YOUR API ACCESS KEY HERE",
+    )
+    // sync way - promise will be resolved with the all the data
+    const syncJob = await client.crawl({
+            "items_limit": 10,
+            "url": "https://stripe.com/",
+            "scrape_type": "markdown"
+        }
     )
-    const response = await client.scrape({
-        input: {
-            "url": "https://www.funda.nl/detail/koop/heerhugowaard/huis-govert-flinckplantsoen-1/89968455/"
-        },
-        crawler_id: "webcrawler/funda",
-    })
-    console.log(response)
+    console.log(syncJob);
+
+    // or async - get the job id and then poll the job status and get the data
+
+    const jobWithId = await client.crawlAsync({
+            "items_limit": 10,
+            "url": "https://stripe.com/",
+            "scrape_type": "markdown"
+        }
+    )
+    // wait for job to complete
+    const jobId = jobWithId.id;
+    let asyncJob = await client.getJob(jobId);
+
+    console.log(asyncJob);
 }
 
 main().catch(console.error);
 ```
 
 ## Response example
-```json
+```javascript
 {
-  city: 'Heerhugowaard',
-  price: 325000,
-  images: [ 'https://cloud.funda.nl/valentina_media/191/215/183_2160.jpg' ],
-  status: 'inonderhandeling',
-  videos: [],
-  address: 'Govert Flinckplantsoen 1',
-  country: 'Nederland',
-  province: 'Noord-Holland',
-  plot_area: '183 m²',
-  post_code: '1701NH',
-  description: 'De woning is met liefde en zorg 53 jaar bewoond door...',
-  living_area: 127,
-  house_number: 1,
-  energie_label: 'd',
-  property_type: 'woonhuis',
-  publication_date: '2024-05-28T00:00:00',
-  number_of_bedrooms: 4,
-  coordinates_latitude: 52.67685,
-  year_of_construction: 1971,
-  coordinates_longitude: 4.8560443,
-  house_number_extension: ''
+  id: '49c4942b-b7d9-4d62-94b5-b54a3016ac51',
+  org_id: 'clxsnorta00075wuuqxgzzvxm',
+  url: 'https://stripe.com/',
+  scrape_type: 'markdown',
+  whitelist_regexp: '',
+  blacklist_regexp: '',
+  allow_subdomains: false,
+  items_limit: 10,
+  created_at: '2024-12-28T21:36:04.417Z',
+  finished_at: null,
+  updated_at: '2024-12-28T21:36:04.383Z',
+  webhook_url: '',
+  status: 'in_progress',
+  job_items: [
+    {
+      id: 'f26cefe1-09d1-4d4c-8b74-b65e075e230d',
+      job_id: '49c4942a-b7d9-4d62-94b5-b54a3016ac51',
+      original_url: 'https://stripe.com/',
+      page_status_code: 0,
+      status: 'new',
+      title: '',
+      last_error: '',
+      created_at: '2024-12-28T21:36:04.468Z',
+      updated_at: '2024-12-28T21:36:04.435Z',
+      cost: 0,
+      referred_url: ''
+    }
+  ],
+  recommended_pull_delay_ms: 5000
 }
 ```

package/api.ts

@@ -1,4 +1,4 @@
-import {ScrapeIDResponse, ScrapeRequest, ScrapeResponse} from "./model";
+import {CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse} from "./model";
 
 const BASE_PATH = "https://api.webcrawlerapi.com"
 const initialPullDelayMs = 2000
@@ -15,14 +15,15 @@ export class WebcrawlerClient {
         this.apiVersion = apiVersion;
     }
 
-    public async scrapeAsync(scrapeRequest: ScrapeRequest): Promise<ScrapeIDResponse> {
+    public async scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId> {
         const url = `${this.basePath}/${this.apiVersion}/scrape`;
 
         const requestOptions = {
             'method': 'POST',
             'headers': {
                 'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
             },
             'body': JSON.stringify(scrapeRequest),
         };
@@ -41,7 +42,7 @@ export class WebcrawlerClient {
         }
     }
 
-    public async scrapeWithMeta(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse> {
+    public async scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<ScrapeResponse> {
         const url = `${this.basePath}/${this.apiVersion}/scrape`;
 
         const requestOptions = {
@@ -53,16 +54,16 @@ export class WebcrawlerClient {
             'body': JSON.stringify(scrapeRequest),
         };
 
-        const scrapeIDResponse: ScrapeIDResponse = await this.sendRequest(url, requestOptions);
+        const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
 
-        if (scrapeIDResponse.id === '') {
+        if (jobIdResponse.id === '') {
             throw new Error("Failed to fetch job status");
         }
 
         let delayIntervalMs = initialPullDelayMs;
-        for (let i = 0; i < MaxPullRetries; i++) {
+        for (let i = 0; i < maxPollingRetries; i++) {
             await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-            const scrapeResult = await this.getScrapeResult(scrapeIDResponse.id);
+            const scrapeResult = await this.getScrapeResult(jobIdResponse.id);
             if (scrapeRequest.debug) {
                 console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
             }
@@ -73,11 +74,11 @@ export class WebcrawlerClient {
                 delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
             }
         }
-        throw new Error("Scraping took too long, please retry");
+        throw new Error("Scraping took too long, please retry or increase the number of polling retries");
     }
 
-    public async scrape(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse> {
-        const scrapeResult = await this.scrapeWithMeta(scrapeRequest);
+    public async scrape(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<any> {
+        const scrapeResult = await this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
         return scrapeResult.structured_data;
     }
 
@@ -87,8 +88,83 @@ export class WebcrawlerClient {
             'method': 'GET',
             'headers': {
                 'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+            },
+        };
+        const response = await fetch(url, requestOptions);
+        if (response.ok) {
+            return response.json();
+        }
+
+        try {
+            const data = await response.json();
+            throw new Error(
+                `failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`
+            );
+        } catch (e) {
+            throw new Error(
+                `failed to fetch job status ${response.status} ${response.statusText}`
+            );
+        }
+    }
+
+    public async crawl(crawlRequest: CrawlRequest): Promise<Job> {
+        const url = `${this.basePath}/${this.apiVersion}/crawl`;
+
+        const requestOptions = {
+            'method': 'POST',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+            },
+            'body': JSON.stringify(crawlRequest),
+        };
+
+        const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
+
+        if (jobIdResponse.id === '') {
+            throw new Error("Failed to fetch job status");
+        }
+
+        let delayIntervalMs = initialPullDelayMs;
+        for (let i = 0; i < MaxPullRetries; i++) {
+            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
+            const job = await this.getJob(jobIdResponse.id);
+            if (job.status !== 'in_progress' && job.status !== 'new') {
+                return job;
+            }
+            if (job.recommended_pull_delay_ms > 0) {
+                delayIntervalMs = job.recommended_pull_delay_ms;
+            }
+        }
+    }
+
+    public async crawlAsync(crawlRequest: CrawlRequest): Promise<JobId> {
+        const url = `${this.basePath}/${this.apiVersion}/crawl`;
+
+        const requestOptions = {
+            'method': 'POST',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
             },
+            'body': JSON.stringify(crawlRequest),
         };
+
+        return await this.sendRequest(url, requestOptions);
+    }
+
+    public async getJob(jobID: string): Promise<Job> {
+        const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+            }
+        }
         const response = await fetch(url, requestOptions);
         if (response.ok) {
             return response.json();
@@ -106,6 +182,7 @@ export class WebcrawlerClient {
         }
     }
 
+
     private async sendRequest(url: string, requestOptions: any): Promise<any> {
         let response: Response;
         try {

package/dist/api.d.ts

@@ -1,12 +1,15 @@
-import { ScrapeIDResponse, ScrapeRequest, ScrapeResponse } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse } from "./model";
 export declare class WebcrawlerClient {
     protected apiVersion: string;
     protected apiKey: string;
     protected basePath: string;
     constructor(apiKey: string, basePath?: string, apiVersion?: string);
-    scrapeAsync(scrapeRequest: ScrapeRequest): Promise<ScrapeIDResponse>;
+    scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
     scrapeWithMeta(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse>;
     scrape(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse>;
     getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
+    crawl(crawlRequest: CrawlRequest): Promise<Job>;
+    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
+    getJob(jobID: string): Promise<Job>;
     private sendRequest;
 }

package/dist/api.js

@@ -55,14 +55,14 @@ class WebcrawlerClient {
                 },
                 'body': JSON.stringify(scrapeRequest),
             };
-            const scrapeIDResponse = yield this.sendRequest(url, requestOptions);
-            if (scrapeIDResponse.id === '') {
+            const jobIdResponse = yield this.sendRequest(url, requestOptions);
+            if (jobIdResponse.id === '') {
                 throw new Error("Failed to fetch job status");
             }
             let delayIntervalMs = initialPullDelayMs;
             for (let i = 0; i < MaxPullRetries; i++) {
                 yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-                const scrapeResult = yield this.getScrapeResult(scrapeIDResponse.id);
+                const scrapeResult = yield this.getScrapeResult(jobIdResponse.id);
                 if (scrapeRequest.debug) {
                     console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
                 }
@@ -104,6 +104,71 @@ class WebcrawlerClient {
             }
         });
     }
+    crawl(crawlRequest) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.basePath}/${this.apiVersion}/crawl`;
+            const requestOptions = {
+                'method': 'POST',
+                'headers': {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this.apiKey}`
+                },
+                'body': JSON.stringify(crawlRequest),
+            };
+            const jobIdResponse = yield this.sendRequest(url, requestOptions);
+            if (jobIdResponse.id === '') {
+                throw new Error("Failed to fetch job status");
+            }
+            let delayIntervalMs = initialPullDelayMs;
+            for (let i = 0; i < MaxPullRetries; i++) {
+                yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
+                const job = yield this.getJob(jobIdResponse.id);
+                if (job.status !== 'in_progress' && job.status !== 'new') {
+                    return job;
+                }
+                if (job.recommended_pull_delay_ms > 0) {
+                    delayIntervalMs = job.recommended_pull_delay_ms;
+                }
+            }
+        });
+    }
+    crawlAsync(crawlRequest) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.basePath}/${this.apiVersion}/crawl`;
+            const requestOptions = {
+                'method': 'POST',
+                'headers': {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this.apiKey}`
+                },
+                'body': JSON.stringify(crawlRequest),
+            };
+            return yield this.sendRequest(url, requestOptions);
+        });
+    }
+    getJob(jobID) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
+            const requestOptions = {
+                'method': 'GET',
+                'headers': {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this.apiKey}`
+                }
+            };
+            const response = yield fetch(url, requestOptions);
+            if (response.ok) {
+                return response.json();
+            }
+            try {
+                const data = yield response.json();
+                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`);
+            }
+            catch (e) {
+                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}`);
+            }
+        });
+    }
     sendRequest(url, requestOptions) {
         return __awaiter(this, void 0, void 0, function* () {
             let response;

package/dist/model.d.ts

@@ -4,7 +4,7 @@ export interface ScrapeRequest {
     max_retries?: number;
     debug?: boolean;
 }
-export interface ScrapeIDResponse {
+export interface JobId {
     id: string;
 }
 export interface ScrapeResponse {
@@ -16,3 +16,43 @@ export interface ScrapeResponse {
     structured_data: any;
     recommended_pull_delay_ms: number;
 }
+export interface CrawlRequest {
+    url: string;
+    items_limit?: number;
+    scrape_type?: string;
+    whitelist_regexp?: string;
+    blacklist_regexp?: string;
+    allow_subdomains?: boolean;
+    webhook_url?: string;
+}
+export interface Job {
+    id: string;
+    org_id: string;
+    url: string;
+    status: string;
+    scrape_type: string;
+    whitelist_regexp: string;
+    blacklist_regexp: string;
+    allow_subdomains: boolean;
+    items_limit: number;
+    created_at: string;
+    finished_at: string;
+    updated_at: string;
+    webhook_url: string;
+    recommended_pull_delay_ms: number;
+    job_items: JobItem[];
+}
+export interface JobItem {
+    id: string;
+    job_id: string;
+    original_url: string;
+    page_status_code: number;
+    markdown_content_url: string;
+    status: string;
+    title: string;
+    last_error: string;
+    created_at: string;
+    updated_at: string;
+    cost: number;
+    referred_url: string;
+}

package/model.ts

@@ -5,7 +5,7 @@ export interface ScrapeRequest {
     debug?: boolean;
 }
 
-export interface ScrapeIDResponse {
+export interface JobId {
     id: string;
 }
 
@@ -18,3 +18,46 @@ export interface ScrapeResponse {
     structured_data: any;
     recommended_pull_delay_ms: number;
 }
+
+export interface CrawlRequest {
+    url: string;
+    items_limit?: number;
+    scrape_type?: string;
+    whitelist_regexp?: string;
+    blacklist_regexp?: string;
+    allow_subdomains?: boolean;
+    webhook_url?: string;
+}
+
+export interface Job {
+    id: string;
+    org_id: string;
+    url: string;
+    status: string;
+    scrape_type: string;
+    whitelist_regexp: string;
+    blacklist_regexp: string;
+    allow_subdomains: boolean;
+    items_limit: number;
+    created_at: string;
+    finished_at: string;
+    updated_at: string;
+    webhook_url: string;
+    recommended_pull_delay_ms: number;
+    job_items: JobItem[];
+}
+
+export interface JobItem {
+    id: string;
+    job_id: string;
+    original_url: string;
+    page_status_code: number;
+    markdown_content_url: string;
+    status: string;
+    title: string;
+    last_error: string;
+    created_at: string;
+    updated_at: string;
+    cost: number;
+    referred_url: string;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "JS client for WecrawlerAPI",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -17,7 +17,7 @@
     "@types/node": "^22.4.0",
     "typescript": "^5.5.4"
   },
-  "author": "Andrew <support@webcrawlerapi.com>",
+  "author": "Andrew <sdk@webcrawlerapi.com>",
   "license": "MIT",
   "dependencies": {},
   "scripts": {