webcrawlerapi-js 1.0.4 → 1.0.5

package/README.md

@@ -1,60 +1,85 @@
-# JS client for WebcrawlerAPI scrapers
+# JS client for WebcrawlerAPI
 
-Official client for [WebcrawlerAPI](https://webcrawlerapi.com/) scrapers.
+Official client for [WebcrawlerAPI](https://webcrawlerapi.com/).
 
-## Installation
-Install WebcrawlerAPI js package:
-
-`npm i webcrawlerapi-js`
+WebcrawlerAPI allows you to extract data from any website with just a simple API call.
 
 ## Preparation
 1. Register to [dashboard|https://dash.webcrawlerapi.com/].
 2. Get an [Access Key](https://dash.webcrawlerapi.com/access).
 
+## Installation
+Install WebcrawlerAPI js package:
+
+`npm i webcrawlerapi-js`
+
 ## Request example
 
 ```javascript
-const webcrawlerapi = require('webcrawlerapi-js');
+import webcrawlerapi from "webcrawlerapi-js";
 
 async function main() {
     const client = new webcrawlerapi.WebcrawlerClient(
-        "YOUR API KEY HERE"
+        "YOUR API ACCESS KEY HERE",
+    )
+    // sync way - promise will be resolved with the all the data
+    const syncJob = await client.crawl({
+            "items_limit": 10,
+            "url": "https://stripe.com/",
+            "scrape_type": "markdown"
+        }
     )
-    const response = await client.scrape({
-        input: {
-            "url": "https://www.funda.nl/detail/koop/heerhugowaard/huis-govert-flinckplantsoen-1/89968455/"
-        },
-        crawler_id: "webcrawler/funda",
-    })
-    console.log(response)
+    console.log(syncJob);
+
+    // or async - get the job id and then poll the job status and get the data
+
+    const jobWithId = await client.crawlAsync({
+            "items_limit": 10,
+            "url": "https://stripe.com/",
+            "scrape_type": "markdown"
+        }
+    )
+    // wait for job to complete
+    const jobId = jobWithId.id;
+    let asyncJob = await client.getJob(jobId);
+
+    console.log(asyncJob);
 }
 
 main().catch(console.error);
 ```
 
 ## Response example
-```json
+```javascript
 {
-  city: 'Heerhugowaard',
-  price: 325000,
-  images: [ 'https://cloud.funda.nl/valentina_media/191/215/183_2160.jpg' ],
-  status: 'inonderhandeling',
-  videos: [],
-  address: 'Govert Flinckplantsoen 1',
-  country: 'Nederland',
-  province: 'Noord-Holland',
-  plot_area: '183 m²',
-  post_code: '1701NH',
-  description: 'De woning is met liefde en zorg 53 jaar bewoond door...',
-  living_area: 127,
-  house_number: 1,
-  energie_label: 'd',
-  property_type: 'woonhuis',
-  publication_date: '2024-05-28T00:00:00',
-  number_of_bedrooms: 4,
-  coordinates_latitude: 52.67685,
-  year_of_construction: 1971,
-  coordinates_longitude: 4.8560443,
-  house_number_extension: ''
+  id: '49c4942b-b7d9-4d62-94b5-b54a3016ac51',
+  org_id: 'clxsnorta00075wuuqxgzzvxm',
+  url: 'https://stripe.com/',
+  scrape_type: 'markdown',
+  whitelist_regexp: '',
+  blacklist_regexp: '',
+  allow_subdomains: false,
+  items_limit: 10,
+  created_at: '2024-12-28T21:36:04.417Z',
+  finished_at: null,
+  updated_at: '2024-12-28T21:36:04.383Z',
+  webhook_url: '',
+  status: 'in_progress',
+  job_items: [
+    {
+      id: 'f26cefe1-09d1-4d4c-8b74-b65e075e230d',
+      job_id: '49c4942a-b7d9-4d62-94b5-b54a3016ac51',
+      original_url: 'https://stripe.com/',
+      page_status_code: 0,
+      status: 'new',
+      title: '',
+      last_error: '',
+      created_at: '2024-12-28T21:36:04.468Z',
+      updated_at: '2024-12-28T21:36:04.435Z',
+      cost: 0,
+      referred_url: ''
+    }
+  ],
+  recommended_pull_delay_ms: 5000
 }
 ```

package/api.ts

@@ -22,7 +22,8 @@ export class WebcrawlerClient {
             'method': 'POST',
             'headers': {
                 'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
             },
             'body': JSON.stringify(scrapeRequest),
         };
@@ -41,7 +42,7 @@ export class WebcrawlerClient {
         }
     }
 
-    public async scrapeWithMeta(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse> {
+    public async scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<ScrapeResponse> {
         const url = `${this.basePath}/${this.apiVersion}/scrape`;
 
         const requestOptions = {
@@ -60,7 +61,7 @@ export class WebcrawlerClient {
         }
 
         let delayIntervalMs = initialPullDelayMs;
-        for (let i = 0; i < MaxPullRetries; i++) {
+        for (let i = 0; i < maxPollingRetries; i++) {
             await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
             const scrapeResult = await this.getScrapeResult(jobIdResponse.id);
             if (scrapeRequest.debug) {
@@ -73,11 +74,11 @@ export class WebcrawlerClient {
                 delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
             }
         }
-        throw new Error("Scraping took too long, please retry");
+        throw new Error("Scraping took too long, please retry or increase the number of polling retries");
     }
 
-    public async scrape(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse> {
-        const scrapeResult = await this.scrapeWithMeta(scrapeRequest);
+    public async scrape(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<any> {
+        const scrapeResult = await this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
         return scrapeResult.structured_data;
     }
 
@@ -87,6 +88,7 @@ export class WebcrawlerClient {
             'method': 'GET',
             'headers': {
                 'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
             },
         };
         const response = await fetch(url, requestOptions);
@@ -113,7 +115,8 @@ export class WebcrawlerClient {
             'method': 'POST',
             'headers': {
                 'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
             },
             'body': JSON.stringify(crawlRequest),
         };
@@ -158,7 +161,8 @@ export class WebcrawlerClient {
             'method': 'GET',
             'headers': {
                 'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
             }
         }
         const response = await fetch(url, requestOptions);

package/dist/api.js

@@ -149,7 +149,6 @@ class WebcrawlerClient {
     getJob(jobID) {
         return __awaiter(this, void 0, void 0, function* () {
             const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
-            console.log(url);
             const requestOptions = {
                 'method': 'GET',
                 'headers': {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "description": "JS client for WecrawlerAPI",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -17,7 +17,7 @@
     "@types/node": "^22.4.0",
     "typescript": "^5.5.4"
   },
-  "author": "Andrew <support@webcrawlerapi.com>",
+  "author": "Andrew <sdk@webcrawlerapi.com>",
   "license": "MIT",
   "dependencies": {},
   "scripts": {