webcrawlerapi-js 1.0.8 → 1.0.10

package/README.md

@@ -16,10 +16,10 @@ Install WebcrawlerAPI js package:
 ## Request example
 
 ```javascript
-import webcrawlerapi from "webcrawlerapi-js";
+import { WebcrawlerClient } from "webcrawlerapi-js"
 
 async function main() {
-    const client = new webcrawlerapi.WebcrawlerClient(
+    const client = new WebcrawlerClient(
         "YOUR API ACCESS KEY HERE",
     )
     // sync way - promise will be resolved with the all the data
@@ -79,6 +79,7 @@ main().catch(console.error);
       status: 'new',
       title: '',
       last_error: '',
+      error_code: '',
       created_at: '2024-12-28T21:36:04.468Z',
       updated_at: '2024-12-28T21:36:04.435Z',
       cost: 0,

package/api.ts

@@ -1,4 +1,6 @@
 import {CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse} from "./model";
+import { JobStatus } from "./constants";
+import { WebcrawlerApiError, createErrorFromResponse } from "./errors";
 
 const BASE_PATH = "https://api.webcrawlerapi.com"
 const initialPullDelayMs = 2000
@@ -28,18 +30,7 @@ export class WebcrawlerClient {
             'body': JSON.stringify(scrapeRequest),
         };
 
-        const response = await fetch(url, requestOptions);
-        if (response.ok) {
-            return response.json();
-        }
-        try {
-            const data = await response.json();
-            throw new Error(
-                `failed to scrape, response returned ${response.status} ${response.statusText}: ${data?.error}`
-            );
-        } catch (e) {
-            throw e;
-        }
+        return await this.sendRequest(url, requestOptions);
     }
 
     public async scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<ScrapeResponse> {
@@ -57,7 +48,7 @@ export class WebcrawlerClient {
         const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
 
         if (jobIdResponse.id === '') {
-            throw new Error("Failed to fetch job status");
+            throw new WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
         }
 
         let delayIntervalMs = initialPullDelayMs;
@@ -67,14 +58,14 @@ export class WebcrawlerClient {
             if (scrapeRequest.debug) {
                 console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
             }
-            if (scrapeResult.status !== 'in_progress' && scrapeResult.status !== 'new') {
+            if (scrapeResult.status !== JobStatus.IN_PROGRESS && scrapeResult.status !== JobStatus.NEW) {
                 return scrapeResult;
             }
             if (scrapeResult.recommended_pull_delay_ms > 0) {
                 delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
             }
         }
-        throw new Error("Scraping took too long, please retry or increase the number of polling retries");
+        throw new WebcrawlerApiError('timeout', 'Scraping took too long, please retry or increase the number of polling retries', 0);
     }
 
     public async scrape(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<any> {
@@ -95,21 +86,7 @@ export class WebcrawlerClient {
                 'Expires': '0'
             },
         };
-        const response = await fetch(url, requestOptions);
-        if (response.ok) {
-            return response.json();
-        }
-
-        try {
-            const data = await response.json();
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`
-            );
-        } catch (e) {
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}`
-            );
-        }
+        return await this.sendRequest(url, requestOptions);
     }
 
     public async crawl(crawlRequest: CrawlRequest): Promise<Job> {
@@ -131,7 +108,7 @@ export class WebcrawlerClient {
         const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
 
         if (jobIdResponse.id === '') {
-            throw new Error("Failed to fetch job status");
+            throw new WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
         }
 
         let delayIntervalMs = initialPullDelayMs;
@@ -139,12 +116,12 @@ export class WebcrawlerClient {
             await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
             const timestamp = new Date().getTime();
             const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
-            if (job.status !== 'in_progress' && job.status !== 'new') {
+            if (job.status !== JobStatus.IN_PROGRESS && job.status !== JobStatus.NEW) {
                 // Transform each job item to include getContent method
                 job.job_items = job.job_items.map(item => ({
                     ...item,
                     getContent: async function(): Promise<string | null> {
-                        if (this.status !== 'done') {
+                        if (job.status !== JobStatus.DONE || this.status !== JobStatus.DONE) {
                             return null;
                         }
 
@@ -185,7 +162,7 @@ export class WebcrawlerClient {
                 delayIntervalMs = job.recommended_pull_delay_ms;
             }
         }
-        throw new Error("Crawling took too long, please retry or increase the number of polling retries");
+        throw new WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
     }
 
     public async crawlAsync(crawlRequest: CrawlRequest): Promise<JobId> {
@@ -215,36 +192,35 @@ export class WebcrawlerClient {
                 'Pragma': 'no-cache',
                 'Expires': '0'
             }
-        }
-        const response = await fetch(url, requestOptions);
-        if (response.ok) {
-            return response.json();
-        }
-
-        try {
-            const data = await response.json();
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`
-            );
-        } catch (e) {
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}`
-            );
-        }
+        };
+        return await this.sendRequest(url, requestOptions);
     }
 
-
     private async sendRequest(url: string, requestOptions: any): Promise<any> {
         let response: Response;
         try {
             response = await fetch(url, requestOptions);
         } catch (e) {
-            throw new Error(`Failed to send request: ${e}`);
+            throw new WebcrawlerApiError('network_error', `Failed to send request: ${e}`, 0);
         }
+
         if (!response.ok) {
-            const errorResponse = await response.json();
-            throw new Error(`${JSON.stringify(errorResponse)}`);
+            try {
+                const errorData = await response.json();
+                throw createErrorFromResponse(response, errorData);
+            } catch (e) {
+                if (e instanceof WebcrawlerApiError) {
+                    throw e;
+                }
+                // If we can't parse the error response, create a generic error
+                throw new WebcrawlerApiError(
+                    'unknown_error',
+                    `Request failed with status ${response.status} ${response.statusText}`,
+                    response.status
+                );
+            }
         }
+
         return response.json();
     }
 }

package/constants.ts

@@ -0,0 +1,14 @@
+export const JobStatus = {
+    IN_PROGRESS: 'in_progress',
+    NEW: 'new',
+    DONE: 'done'
+} as const;
+
+export const HttpStatus = {
+    OK: 200,
+    BAD_REQUEST: 400,
+    UNAUTHORIZED: 401,
+    FORBIDDEN: 403,
+    NOT_FOUND: 404,
+    INTERNAL_SERVER_ERROR: 500
+} as const; 

package/dist/constants.d.ts

@@ -0,0 +1,13 @@
+export declare const JobStatus: {
+    readonly IN_PROGRESS: "in_progress";
+    readonly NEW: "new";
+    readonly DONE: "done";
+};
+export declare const HttpStatus: {
+    readonly OK: 200;
+    readonly BAD_REQUEST: 400;
+    readonly UNAUTHORIZED: 401;
+    readonly FORBIDDEN: 403;
+    readonly NOT_FOUND: 404;
+    readonly INTERNAL_SERVER_ERROR: 500;
+};

package/dist/constants.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpStatus = exports.JobStatus = void 0;
+exports.JobStatus = {
+    IN_PROGRESS: 'in_progress',
+    NEW: 'new',
+    DONE: 'done'
+};
+exports.HttpStatus = {
+    OK: 200,
+    BAD_REQUEST: 400,
+    UNAUTHORIZED: 401,
+    FORBIDDEN: 403,
+    NOT_FOUND: 404,
+    INTERNAL_SERVER_ERROR: 500
+};

package/dist/errors.d.ts

@@ -0,0 +1,12 @@
+export declare class WebcrawlerApiError extends Error {
+    errorCode: string;
+    errorMessage: string;
+    statusCode?: number;
+    constructor(errorCode: string, errorMessage: string, statusCode?: number);
+}
+export interface ErrorResponse {
+    error_code?: string;
+    error_message?: string;
+    error?: string;
+}
+export declare function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError;

package/dist/errors.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebcrawlerApiError = void 0;
+exports.createErrorFromResponse = createErrorFromResponse;
+class WebcrawlerApiError extends Error {
+    constructor(errorCode, errorMessage, statusCode) {
+        super(errorMessage);
+        this.name = 'WebcrawlerApiError';
+        this.errorCode = errorCode;
+        this.errorMessage = errorMessage;
+        this.statusCode = statusCode;
+    }
+}
+exports.WebcrawlerApiError = WebcrawlerApiError;
+function createErrorFromResponse(response, data) {
+    const errorCode = data.error_code || 'unknown_error';
+    const errorMessage = data.error_message || data.error || 'Unknown error';
+    return new WebcrawlerApiError(errorCode, errorMessage, response.status);
+}

package/errors.ts

@@ -0,0 +1,25 @@
+export class WebcrawlerApiError extends Error {
+    public errorCode: string;
+    public errorMessage: string;
+    public statusCode?: number;
+
+    constructor(errorCode: string, errorMessage: string, statusCode?: number) {
+        super(errorMessage);
+        this.name = 'WebcrawlerApiError';
+        this.errorCode = errorCode;
+        this.errorMessage = errorMessage;
+        this.statusCode = statusCode;
+    }
+}
+
+export interface ErrorResponse {
+    error_code?: string;
+    error_message?: string;
+    error?: string;
+}
+
+export function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError {
+    const errorCode = data.error_code || 'unknown_error';
+    const errorMessage = data.error_message || data.error || 'Unknown error';
+    return new WebcrawlerApiError(errorCode, errorMessage, response.status);
+} 

package/index.ts

@@ -1 +1,4 @@
-export * from "./api";
+export * from "./api";
+export * from "./model";
+export * from "./constants";
+export * from "./errors";

package/package.json

@@ -1,9 +1,13 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.0.8",
-  "description": "JS client for WecrawlerAPI",
+  "version": "1.0.10",
+
+  "description": "JS client for WebcrawlerAPI",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
+  "scripts": {
+    "build": "tsc --outDir dist/"
+  },
   "keywords": [
     "scraping",
     "scraper",
@@ -19,8 +23,5 @@
   },
   "author": "Andrew <sdk@webcrawlerapi.com>",
   "license": "MIT",
-  "dependencies": {},
-  "scripts": {
-    "build": "tsc --outDir dist/"
-  }
-}
+  "dependencies": {}
+}

package/dist/api.js

@@ -1,238 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WebcrawlerClient = void 0;
-const BASE_PATH = "https://api.webcrawlerapi.com";
-const initialPullDelayMs = 2000;
-const MaxPullRetries = 100;
-class WebcrawlerClient {
-    constructor(apiKey, basePath = BASE_PATH, apiVersion = "v1") {
-        this.apiVersion = "v1";
-        this.apiKey = apiKey;
-        this.basePath = basePath;
-        this.apiVersion = apiVersion;
-    }
-    scrapeAsync(scrapeRequest) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/scrape`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client"
-                },
-                'body': JSON.stringify(scrapeRequest),
-            };
-            const response = yield fetch(url, requestOptions);
-            if (response.ok) {
-                return response.json();
-            }
-            try {
-                const data = yield response.json();
-                throw new Error(`failed to scrape, response returned ${response.status} ${response.statusText}: ${data === null || data === void 0 ? void 0 : data.error}`);
-            }
-            catch (e) {
-                throw e;
-            }
-        });
-    }
-    scrapeWithMeta(scrapeRequest_1) {
-        return __awaiter(this, arguments, void 0, function* (scrapeRequest, maxPollingRetries = MaxPullRetries) {
-            const url = `${this.basePath}/${this.apiVersion}/scrape`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`
-                },
-                'body': JSON.stringify(scrapeRequest),
-            };
-            const jobIdResponse = yield this.sendRequest(url, requestOptions);
-            if (jobIdResponse.id === '') {
-                throw new Error("Failed to fetch job status");
-            }
-            let delayIntervalMs = initialPullDelayMs;
-            for (let i = 0; i < maxPollingRetries; i++) {
-                yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-                const scrapeResult = yield this.getScrapeResult(jobIdResponse.id);
-                if (scrapeRequest.debug) {
-                    console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
-                }
-                if (scrapeResult.status !== 'in_progress' && scrapeResult.status !== 'new') {
-                    return scrapeResult;
-                }
-                if (scrapeResult.recommended_pull_delay_ms > 0) {
-                    delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
-                }
-            }
-            throw new Error("Scraping took too long, please retry or increase the number of polling retries");
-        });
-    }
-    scrape(scrapeRequest_1) {
-        return __awaiter(this, arguments, void 0, function* (scrapeRequest, maxPollingRetries = MaxPullRetries) {
-            const scrapeResult = yield this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
-            return scrapeResult.structured_data;
-        });
-    }
-    getScrapeResult(scrapeID) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
-            const requestOptions = {
-                'method': 'GET',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                    'Cache-Control': 'no-cache, no-store, must-revalidate',
-                    'Pragma': 'no-cache',
-                    'Expires': '0'
-                },
-            };
-            const response = yield fetch(url, requestOptions);
-            if (response.ok) {
-                return response.json();
-            }
-            try {
-                const data = yield response.json();
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`);
-            }
-            catch (e) {
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}`);
-            }
-        });
-    }
-    crawl(crawlRequest) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/crawl`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                    'Cache-Control': 'no-cache, no-store, must-revalidate',
-                    'Pragma': 'no-cache',
-                    'Expires': '0'
-                },
-                'body': JSON.stringify(crawlRequest),
-            };
-            const jobIdResponse = yield this.sendRequest(url, requestOptions);
-            if (jobIdResponse.id === '') {
-                throw new Error("Failed to fetch job status");
-            }
-            let delayIntervalMs = initialPullDelayMs;
-            for (let i = 0; i < MaxPullRetries; i++) {
-                yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-                const timestamp = new Date().getTime();
-                const job = yield this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
-                if (job.status !== 'in_progress' && job.status !== 'new') {
-                    // Transform each job item to include getContent method
-                    job.job_items = job.job_items.map(item => (Object.assign(Object.assign({}, item), { getContent: function () {
-                            return __awaiter(this, void 0, void 0, function* () {
-                                if (this.status !== 'done') {
-                                    return null;
-                                }
-                                let contentUrl;
-                                switch (job.scrape_type) {
-                                    case 'html':
-                                        contentUrl = this.raw_content_url;
-                                        break;
-                                    case 'cleaned':
-                                        contentUrl = this.cleaned_content_url;
-                                        break;
-                                    case 'markdown':
-                                        contentUrl = this.markdown_content_url;
-                                        break;
-                                }
-                                if (!contentUrl) {
-                                    return null;
-                                }
-                                const response = yield fetch(contentUrl, {
-                                    headers: {
-                                        'Accept-Encoding': 'gzip, deflate, br',
-                                        'Accept': '*/*'
-                                    }
-                                });
-                                if (!response.ok) {
-                                    throw new Error(`Failed to fetch content: ${response.statusText}`);
-                                }
-                                return yield response.text();
-                            });
-                        } })));
-                    return job;
-                }
-                if (job.recommended_pull_delay_ms > 0) {
-                    delayIntervalMs = job.recommended_pull_delay_ms;
-                }
-            }
-            throw new Error("Crawling took too long, please retry or increase the number of polling retries");
-        });
-    }
-    crawlAsync(crawlRequest) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/crawl`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`
-                },
-                'body': JSON.stringify(crawlRequest),
-            };
-            return yield this.sendRequest(url, requestOptions);
-        });
-    }
-    getJob(jobID) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
-            const requestOptions = {
-                'method': 'GET',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                    'Cache-Control': 'no-cache, no-store, must-revalidate',
-                    'Pragma': 'no-cache',
-                    'Expires': '0'
-                }
-            };
-            const response = yield fetch(url, requestOptions);
-            if (response.ok) {
-                return response.json();
-            }
-            try {
-                const data = yield response.json();
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`);
-            }
-            catch (e) {
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}`);
-            }
-        });
-    }
-    sendRequest(url, requestOptions) {
-        return __awaiter(this, void 0, void 0, function* () {
-            let response;
-            try {
-                response = yield fetch(url, requestOptions);
-            }
-            catch (e) {
-                throw new Error(`Failed to send request: ${e}`);
-            }
-            if (!response.ok) {
-                const errorResponse = yield response.json();
-                throw new Error(`${JSON.stringify(errorResponse)}`);
-            }
-            return response.json();
-        });
-    }
-}
-exports.WebcrawlerClient = WebcrawlerClient;

package/dist/index.d.ts

@@ -1 +0,0 @@
-export * from "./api";

package/dist/index.js

@@ -1,17 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./api"), exports);