webcrawlerapi-js 1.0.7 → 1.0.9

package/README.md

@@ -16,10 +16,10 @@ Install WebcrawlerAPI js package:
 ## Request example
 
 ```javascript
-import webcrawlerapi from "webcrawlerapi-js";
+import { WebcrawlerClient } from "webcrawlerapi-js"
 
 async function main() {
-    const client = new webcrawlerapi.WebcrawlerClient(
+    const client = new WebcrawlerClient(
         "YOUR API ACCESS KEY HERE",
     )
     // sync way - promise will be resolved with the all the data
@@ -29,6 +29,11 @@ async function main() {
             "scrape_type": "markdown"
         }
     )
+    for (const item of syncJob.job_items) {
+        item.getContent().then((content) => {
+            console.log(content.slice(0, 100));
+        })
+    }
     console.log(syncJob);
 
     // or async - get the job id and then poll the job status and get the data
@@ -74,6 +79,7 @@ main().catch(console.error);
       status: 'new',
       title: '',
       last_error: '',
+      error_code: '',
       created_at: '2024-12-28T21:36:04.468Z',
       updated_at: '2024-12-28T21:36:04.435Z',
       cost: 0,

package/api.ts

@@ -1,4 +1,5 @@
 import {CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse} from "./model";
+import { JobStatus } from "./constants";
 
 const BASE_PATH = "https://api.webcrawlerapi.com"
 const initialPullDelayMs = 2000
@@ -67,7 +68,7 @@ export class WebcrawlerClient {
             if (scrapeRequest.debug) {
                 console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
             }
-            if (scrapeResult.status !== 'in_progress' && scrapeResult.status !== 'new') {
+            if (scrapeResult.status !== JobStatus.IN_PROGRESS && scrapeResult.status !== JobStatus.NEW) {
                 return scrapeResult;
             }
             if (scrapeResult.recommended_pull_delay_ms > 0) {
@@ -87,8 +88,12 @@ export class WebcrawlerClient {
         const requestOptions = {
             'method': 'GET',
             'headers': {
+                'Content-Type': 'application/json',
                 'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                'Cache-Control': 'no-cache, no-store, must-revalidate',
+                'Pragma': 'no-cache',
+                'Expires': '0'
             },
         };
         const response = await fetch(url, requestOptions);
@@ -116,7 +121,10 @@ export class WebcrawlerClient {
             'headers': {
                 'Content-Type': 'application/json',
                 'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                'Cache-Control': 'no-cache, no-store, must-revalidate',
+                'Pragma': 'no-cache',
+                'Expires': '0'
             },
             'body': JSON.stringify(crawlRequest),
         };
@@ -130,14 +138,55 @@ export class WebcrawlerClient {
         let delayIntervalMs = initialPullDelayMs;
         for (let i = 0; i < MaxPullRetries; i++) {
             await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-            const job = await this.getJob(jobIdResponse.id);
-            if (job.status !== 'in_progress' && job.status !== 'new') {
+            const timestamp = new Date().getTime();
+            const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
+            if (job.status !== JobStatus.IN_PROGRESS && job.status !== JobStatus.NEW) {
+                // Transform each job item to include getContent method
+                job.job_items = job.job_items.map(item => ({
+                    ...item,
+                    getContent: async function(): Promise<string | null> {
+                        if (job.status !== JobStatus.DONE || this.status !== JobStatus.DONE) {
+                            return null;
+                        }
+
+                        let contentUrl: string | undefined;
+                        switch (job.scrape_type) {
+                            case 'html':
+                                contentUrl = this.raw_content_url;
+                                break;
+                            case 'cleaned':
+                                contentUrl = this.cleaned_content_url;
+                                break;
+                            case 'markdown':
+                                contentUrl = this.markdown_content_url;
+                                break;
+                        }
+
+                        if (!contentUrl) {
+                            return null;
+                        }
+
+                        const response = await fetch(contentUrl, {
+                            headers: {
+                                'Accept-Encoding': 'gzip, deflate, br',
+                                'Accept': '*/*'
+                            }
+                        });
+
+                        if (!response.ok) {
+                            throw new Error(`Failed to fetch content: ${response.statusText}`);
+                        }
+
+                        return await response.text();
+                    }
+                }));
                 return job;
             }
             if (job.recommended_pull_delay_ms > 0) {
                 delayIntervalMs = job.recommended_pull_delay_ms;
             }
         }
+        throw new Error("Crawling took too long, please retry or increase the number of polling retries");
     }
 
     public async crawlAsync(crawlRequest: CrawlRequest): Promise<JobId> {
@@ -162,7 +211,10 @@ export class WebcrawlerClient {
             'headers': {
                 'Content-Type': 'application/json',
                 'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                'Cache-Control': 'no-cache, no-store, must-revalidate',
+                'Pragma': 'no-cache',
+                'Expires': '0'
             }
         }
         const response = await fetch(url, requestOptions);

package/constants.ts

@@ -0,0 +1,14 @@
+export const JobStatus = {
+    IN_PROGRESS: 'in_progress',
+    NEW: 'new',
+    DONE: 'done'
+} as const;
+
+export const HttpStatus = {
+    OK: 200,
+    BAD_REQUEST: 400,
+    UNAUTHORIZED: 401,
+    FORBIDDEN: 403,
+    NOT_FOUND: 404,
+    INTERNAL_SERVER_ERROR: 500
+} as const; 

package/dist/api.js

@@ -10,6 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WebcrawlerClient = void 0;
+const constants_1 = require("./constants");
 const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
@@ -67,7 +68,7 @@ class WebcrawlerClient {
                 if (scrapeRequest.debug) {
                     console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
                 }
-                if (scrapeResult.status !== 'in_progress' && scrapeResult.status !== 'new') {
+                if (scrapeResult.status !== constants_1.JobStatus.IN_PROGRESS && scrapeResult.status !== constants_1.JobStatus.NEW) {
                     return scrapeResult;
                 }
                 if (scrapeResult.recommended_pull_delay_ms > 0) {
@@ -89,8 +90,12 @@ class WebcrawlerClient {
             const requestOptions = {
                 'method': 'GET',
                 'headers': {
+                    'Content-Type': 'application/json',
                     'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                    'Cache-Control': 'no-cache, no-store, must-revalidate',
+                    'Pragma': 'no-cache',
+                    'Expires': '0'
                 },
             };
             const response = yield fetch(url, requestOptions);
@@ -114,7 +119,10 @@ class WebcrawlerClient {
                 'headers': {
                     'Content-Type': 'application/json',
                     'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                    'Cache-Control': 'no-cache, no-store, must-revalidate',
+                    'Pragma': 'no-cache',
+                    'Expires': '0'
                 },
                 'body': JSON.stringify(crawlRequest),
             };
@@ -125,14 +133,49 @@ class WebcrawlerClient {
             let delayIntervalMs = initialPullDelayMs;
             for (let i = 0; i < MaxPullRetries; i++) {
                 yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-                const job = yield this.getJob(jobIdResponse.id);
-                if (job.status !== 'in_progress' && job.status !== 'new') {
+                const timestamp = new Date().getTime();
+                const job = yield this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
+                if (job.status !== constants_1.JobStatus.IN_PROGRESS && job.status !== constants_1.JobStatus.NEW) {
+                    // Transform each job item to include getContent method
+                    job.job_items = job.job_items.map(item => (Object.assign(Object.assign({}, item), { getContent: function () {
+                            return __awaiter(this, void 0, void 0, function* () {
+                                if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
+                                    return null;
+                                }
+                                let contentUrl;
+                                switch (job.scrape_type) {
+                                    case 'html':
+                                        contentUrl = this.raw_content_url;
+                                        break;
+                                    case 'cleaned':
+                                        contentUrl = this.cleaned_content_url;
+                                        break;
+                                    case 'markdown':
+                                        contentUrl = this.markdown_content_url;
+                                        break;
+                                }
+                                if (!contentUrl) {
+                                    return null;
+                                }
+                                const response = yield fetch(contentUrl, {
+                                    headers: {
+                                        'Accept-Encoding': 'gzip, deflate, br',
+                                        'Accept': '*/*'
+                                    }
+                                });
+                                if (!response.ok) {
+                                    throw new Error(`Failed to fetch content: ${response.statusText}`);
+                                }
+                                return yield response.text();
+                            });
+                        } })));
                     return job;
                 }
                 if (job.recommended_pull_delay_ms > 0) {
                     delayIntervalMs = job.recommended_pull_delay_ms;
                 }
             }
+            throw new Error("Crawling took too long, please retry or increase the number of polling retries");
         });
     }
     crawlAsync(crawlRequest) {
@@ -157,7 +200,10 @@ class WebcrawlerClient {
                 'headers': {
                     'Content-Type': 'application/json',
                     'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                    'Cache-Control': 'no-cache, no-store, must-revalidate',
+                    'Pragma': 'no-cache',
+                    'Expires': '0'
                 }
             };
             const response = yield fetch(url, requestOptions);

package/dist/constants.d.ts

@@ -0,0 +1,13 @@
+export declare const JobStatus: {
+    readonly IN_PROGRESS: "in_progress";
+    readonly NEW: "new";
+    readonly DONE: "done";
+};
+export declare const HttpStatus: {
+    readonly OK: 200;
+    readonly BAD_REQUEST: 400;
+    readonly UNAUTHORIZED: 401;
+    readonly FORBIDDEN: 403;
+    readonly NOT_FOUND: 404;
+    readonly INTERNAL_SERVER_ERROR: 500;
+};

package/dist/constants.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpStatus = exports.JobStatus = void 0;
+exports.JobStatus = {
+    IN_PROGRESS: 'in_progress',
+    NEW: 'new',
+    DONE: 'done'
+};
+exports.HttpStatus = {
+    OK: 200,
+    BAD_REQUEST: 400,
+    UNAUTHORIZED: 401,
+    FORBIDDEN: 403,
+    NOT_FOUND: 404,
+    INTERNAL_SERVER_ERROR: 500
+};

package/dist/model.d.ts

@@ -56,4 +56,7 @@ export interface JobItem {
     updated_at: string;
     cost: number;
     referred_url: string;
+    raw_content_url?: string;
+    cleaned_content_url?: string;
+    getContent(): Promise<string | null>;
 }

package/model.ts

@@ -61,4 +61,7 @@ export interface JobItem {
     updated_at: string;
     cost: number;
     referred_url: string;
+    raw_content_url?: string;
+    cleaned_content_url?: string;
+    getContent(): Promise<string | null>;
 }

package/package.json

@@ -1,9 +1,13 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.0.7",
+  "version": "1.0.9",
+
   "description": "JS client for WecrawlerAPI",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
+  "scripts": {
+    "build": "tsc --outDir dist/"
+  },
   "keywords": [
     "scraping",
     "scraper",
@@ -19,8 +23,5 @@
   },
   "author": "Andrew <sdk@webcrawlerapi.com>",
   "license": "MIT",
-  "dependencies": {},
-  "scripts": {
-    "build": "tsc --outDir dist/"
-  }
-}
+  "dependencies": {}
+}