webcrawlerapi-js 1.0.9 → 1.0.10

package/api.ts

@@ -1,5 +1,6 @@
 import {CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse} from "./model";
 import { JobStatus } from "./constants";
+import { WebcrawlerApiError, createErrorFromResponse } from "./errors";
 
 const BASE_PATH = "https://api.webcrawlerapi.com"
 const initialPullDelayMs = 2000
@@ -29,18 +30,7 @@ export class WebcrawlerClient {
             'body': JSON.stringify(scrapeRequest),
         };
 
-        const response = await fetch(url, requestOptions);
-        if (response.ok) {
-            return response.json();
-        }
-        try {
-            const data = await response.json();
-            throw new Error(
-                `failed to scrape, response returned ${response.status} ${response.statusText}: ${data?.error}`
-            );
-        } catch (e) {
-            throw e;
-        }
+        return await this.sendRequest(url, requestOptions);
     }
 
     public async scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<ScrapeResponse> {
@@ -58,7 +48,7 @@ export class WebcrawlerClient {
         const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
 
         if (jobIdResponse.id === '') {
-            throw new Error("Failed to fetch job status");
+            throw new WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
         }
 
         let delayIntervalMs = initialPullDelayMs;
@@ -75,7 +65,7 @@ export class WebcrawlerClient {
                 delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
             }
         }
-        throw new Error("Scraping took too long, please retry or increase the number of polling retries");
+        throw new WebcrawlerApiError('timeout', 'Scraping took too long, please retry or increase the number of polling retries', 0);
     }
 
     public async scrape(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<any> {
@@ -96,21 +86,7 @@ export class WebcrawlerClient {
                 'Expires': '0'
             },
         };
-        const response = await fetch(url, requestOptions);
-        if (response.ok) {
-            return response.json();
-        }
-
-        try {
-            const data = await response.json();
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`
-            );
-        } catch (e) {
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}`
-            );
-        }
+        return await this.sendRequest(url, requestOptions);
     }
 
     public async crawl(crawlRequest: CrawlRequest): Promise<Job> {
@@ -132,7 +108,7 @@ export class WebcrawlerClient {
         const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
 
         if (jobIdResponse.id === '') {
-            throw new Error("Failed to fetch job status");
+            throw new WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
         }
 
         let delayIntervalMs = initialPullDelayMs;
@@ -186,7 +162,7 @@ export class WebcrawlerClient {
                 delayIntervalMs = job.recommended_pull_delay_ms;
             }
         }
-        throw new Error("Crawling took too long, please retry or increase the number of polling retries");
+        throw new WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
     }
 
     public async crawlAsync(crawlRequest: CrawlRequest): Promise<JobId> {
@@ -216,36 +192,35 @@ export class WebcrawlerClient {
                 'Pragma': 'no-cache',
                 'Expires': '0'
             }
-        }
-        const response = await fetch(url, requestOptions);
-        if (response.ok) {
-            return response.json();
-        }
-
-        try {
-            const data = await response.json();
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`
-            );
-        } catch (e) {
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}`
-            );
-        }
+        };
+        return await this.sendRequest(url, requestOptions);
     }
 
-
     private async sendRequest(url: string, requestOptions: any): Promise<any> {
         let response: Response;
         try {
             response = await fetch(url, requestOptions);
         } catch (e) {
-            throw new Error(`Failed to send request: ${e}`);
+            throw new WebcrawlerApiError('network_error', `Failed to send request: ${e}`, 0);
         }
+
         if (!response.ok) {
-            const errorResponse = await response.json();
-            throw new Error(`${JSON.stringify(errorResponse)}`);
+            try {
+                const errorData = await response.json();
+                throw createErrorFromResponse(response, errorData);
+            } catch (e) {
+                if (e instanceof WebcrawlerApiError) {
+                    throw e;
+                }
+                // If we can't parse the error response, create a generic error
+                throw new WebcrawlerApiError(
+                    'unknown_error',
+                    `Request failed with status ${response.status} ${response.statusText}`,
+                    response.status
+                );
+            }
         }
+
         return response.json();
     }
 }

package/dist/errors.d.ts

@@ -0,0 +1,12 @@
+export declare class WebcrawlerApiError extends Error {
+    errorCode: string;
+    errorMessage: string;
+    statusCode?: number;
+    constructor(errorCode: string, errorMessage: string, statusCode?: number);
+}
+export interface ErrorResponse {
+    error_code?: string;
+    error_message?: string;
+    error?: string;
+}
+export declare function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError;

package/dist/errors.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebcrawlerApiError = void 0;
+exports.createErrorFromResponse = createErrorFromResponse;
+class WebcrawlerApiError extends Error {
+    constructor(errorCode, errorMessage, statusCode) {
+        super(errorMessage);
+        this.name = 'WebcrawlerApiError';
+        this.errorCode = errorCode;
+        this.errorMessage = errorMessage;
+        this.statusCode = statusCode;
+    }
+}
+exports.WebcrawlerApiError = WebcrawlerApiError;
+function createErrorFromResponse(response, data) {
+    const errorCode = data.error_code || 'unknown_error';
+    const errorMessage = data.error_message || data.error || 'Unknown error';
+    return new WebcrawlerApiError(errorCode, errorMessage, response.status);
+}

package/errors.ts

@@ -0,0 +1,25 @@
+export class WebcrawlerApiError extends Error {
+    public errorCode: string;
+    public errorMessage: string;
+    public statusCode?: number;
+
+    constructor(errorCode: string, errorMessage: string, statusCode?: number) {
+        super(errorMessage);
+        this.name = 'WebcrawlerApiError';
+        this.errorCode = errorCode;
+        this.errorMessage = errorMessage;
+        this.statusCode = statusCode;
+    }
+}
+
+export interface ErrorResponse {
+    error_code?: string;
+    error_message?: string;
+    error?: string;
+}
+
+export function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError {
+    const errorCode = data.error_code || 'unknown_error';
+    const errorMessage = data.error_message || data.error || 'Unknown error';
+    return new WebcrawlerApiError(errorCode, errorMessage, response.status);
+} 

package/index.ts

@@ -1 +1,4 @@
-export * from "./api";
+export * from "./api";
+export * from "./model";
+export * from "./constants";
+export * from "./errors";

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.0.9",
+  "version": "1.0.10",
 
-  "description": "JS client for WecrawlerAPI",
+  "description": "JS client for WebcrawlerAPI",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "scripts": {

package/dist/api.js

@@ -1,239 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WebcrawlerClient = void 0;
-const constants_1 = require("./constants");
-const BASE_PATH = "https://api.webcrawlerapi.com";
-const initialPullDelayMs = 2000;
-const MaxPullRetries = 100;
-class WebcrawlerClient {
-    constructor(apiKey, basePath = BASE_PATH, apiVersion = "v1") {
-        this.apiVersion = "v1";
-        this.apiKey = apiKey;
-        this.basePath = basePath;
-        this.apiVersion = apiVersion;
-    }
-    scrapeAsync(scrapeRequest) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/scrape`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client"
-                },
-                'body': JSON.stringify(scrapeRequest),
-            };
-            const response = yield fetch(url, requestOptions);
-            if (response.ok) {
-                return response.json();
-            }
-            try {
-                const data = yield response.json();
-                throw new Error(`failed to scrape, response returned ${response.status} ${response.statusText}: ${data === null || data === void 0 ? void 0 : data.error}`);
-            }
-            catch (e) {
-                throw e;
-            }
-        });
-    }
-    scrapeWithMeta(scrapeRequest_1) {
-        return __awaiter(this, arguments, void 0, function* (scrapeRequest, maxPollingRetries = MaxPullRetries) {
-            const url = `${this.basePath}/${this.apiVersion}/scrape`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`
-                },
-                'body': JSON.stringify(scrapeRequest),
-            };
-            const jobIdResponse = yield this.sendRequest(url, requestOptions);
-            if (jobIdResponse.id === '') {
-                throw new Error("Failed to fetch job status");
-            }
-            let delayIntervalMs = initialPullDelayMs;
-            for (let i = 0; i < maxPollingRetries; i++) {
-                yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-                const scrapeResult = yield this.getScrapeResult(jobIdResponse.id);
-                if (scrapeRequest.debug) {
-                    console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
-                }
-                if (scrapeResult.status !== constants_1.JobStatus.IN_PROGRESS && scrapeResult.status !== constants_1.JobStatus.NEW) {
-                    return scrapeResult;
-                }
-                if (scrapeResult.recommended_pull_delay_ms > 0) {
-                    delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
-                }
-            }
-            throw new Error("Scraping took too long, please retry or increase the number of polling retries");
-        });
-    }
-    scrape(scrapeRequest_1) {
-        return __awaiter(this, arguments, void 0, function* (scrapeRequest, maxPollingRetries = MaxPullRetries) {
-            const scrapeResult = yield this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
-            return scrapeResult.structured_data;
-        });
-    }
-    getScrapeResult(scrapeID) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
-            const requestOptions = {
-                'method': 'GET',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                    'Cache-Control': 'no-cache, no-store, must-revalidate',
-                    'Pragma': 'no-cache',
-                    'Expires': '0'
-                },
-            };
-            const response = yield fetch(url, requestOptions);
-            if (response.ok) {
-                return response.json();
-            }
-            try {
-                const data = yield response.json();
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`);
-            }
-            catch (e) {
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}`);
-            }
-        });
-    }
-    crawl(crawlRequest) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/crawl`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                    'Cache-Control': 'no-cache, no-store, must-revalidate',
-                    'Pragma': 'no-cache',
-                    'Expires': '0'
-                },
-                'body': JSON.stringify(crawlRequest),
-            };
-            const jobIdResponse = yield this.sendRequest(url, requestOptions);
-            if (jobIdResponse.id === '') {
-                throw new Error("Failed to fetch job status");
-            }
-            let delayIntervalMs = initialPullDelayMs;
-            for (let i = 0; i < MaxPullRetries; i++) {
-                yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-                const timestamp = new Date().getTime();
-                const job = yield this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
-                if (job.status !== constants_1.JobStatus.IN_PROGRESS && job.status !== constants_1.JobStatus.NEW) {
-                    // Transform each job item to include getContent method
-                    job.job_items = job.job_items.map(item => (Object.assign(Object.assign({}, item), { getContent: function () {
-                            return __awaiter(this, void 0, void 0, function* () {
-                                if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
-                                    return null;
-                                }
-                                let contentUrl;
-                                switch (job.scrape_type) {
-                                    case 'html':
-                                        contentUrl = this.raw_content_url;
-                                        break;
-                                    case 'cleaned':
-                                        contentUrl = this.cleaned_content_url;
-                                        break;
-                                    case 'markdown':
-                                        contentUrl = this.markdown_content_url;
-                                        break;
-                                }
-                                if (!contentUrl) {
-                                    return null;
-                                }
-                                const response = yield fetch(contentUrl, {
-                                    headers: {
-                                        'Accept-Encoding': 'gzip, deflate, br',
-                                        'Accept': '*/*'
-                                    }
-                                });
-                                if (!response.ok) {
-                                    throw new Error(`Failed to fetch content: ${response.statusText}`);
-                                }
-                                return yield response.text();
-                            });
-                        } })));
-                    return job;
-                }
-                if (job.recommended_pull_delay_ms > 0) {
-                    delayIntervalMs = job.recommended_pull_delay_ms;
-                }
-            }
-            throw new Error("Crawling took too long, please retry or increase the number of polling retries");
-        });
-    }
-    crawlAsync(crawlRequest) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/crawl`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`
-                },
-                'body': JSON.stringify(crawlRequest),
-            };
-            return yield this.sendRequest(url, requestOptions);
-        });
-    }
-    getJob(jobID) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
-            const requestOptions = {
-                'method': 'GET',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                    'Cache-Control': 'no-cache, no-store, must-revalidate',
-                    'Pragma': 'no-cache',
-                    'Expires': '0'
-                }
-            };
-            const response = yield fetch(url, requestOptions);
-            if (response.ok) {
-                return response.json();
-            }
-            try {
-                const data = yield response.json();
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`);
-            }
-            catch (e) {
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}`);
-            }
-        });
-    }
-    sendRequest(url, requestOptions) {
-        return __awaiter(this, void 0, void 0, function* () {
-            let response;
-            try {
-                response = yield fetch(url, requestOptions);
-            }
-            catch (e) {
-                throw new Error(`Failed to send request: ${e}`);
-            }
-            if (!response.ok) {
-                const errorResponse = yield response.json();
-                throw new Error(`${JSON.stringify(errorResponse)}`);
-            }
-            return response.json();
-        });
-    }
-}
-exports.WebcrawlerClient = WebcrawlerClient;

package/dist/index.d.ts

@@ -1 +0,0 @@
-export * from "./api";

package/dist/index.js

@@ -1,17 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./api"), exports);