webcrawlerapi-js 1.0.9 → 1.1.0

package/README.md

@@ -1,64 +1,190 @@
 # JS client for WebcrawlerAPI
 
-Official client for [WebcrawlerAPI](https://webcrawlerapi.com/).
+Official client for [WebcrawlerAPI](https://webcrawlerapi.com/). This SDK provides a robust and type-safe way to interact with WebcrawlerAPI's powerful web scraping capabilities, making it easy to extract structured data from any website.
 
-WebcrawlerAPI allows you to extract data from any website with just a simple API call.
+WebcrawlerAPI allows you to extract data from any website with just a simple API call. The service handles all the complexities of web scraping, including JavaScript rendering, proxy rotation, and rate limiting, so you can focus on using the data.
 
 ## Preparation
 1. Register to [dashboard|https://dash.webcrawlerapi.com/].
 2. Get an [Access Key](https://dash.webcrawlerapi.com/access).
 
+Our dashboard provides comprehensive analytics and monitoring tools to help you track your API usage and optimize your scraping operations.
+
 ## Installation
 Install WebcrawlerAPI js package:
 
 `npm i webcrawlerapi-js`
 
-## Request example
+The package is regularly updated with new features and improvements to ensure the best possible scraping experience.
+
+## JavaScript Examples
+
+### Synchronous Job
+In this mode, the promise will be resolved when all data is ready. Synchronous jobs are perfect for quick scraping tasks where you need immediate results and don't want to handle polling logic.
 
 ```javascript
-import { WebcrawlerClient } from "webcrawlerapi-js"
+import webcrawlerapi from "webcrawlerapi-js"
 
 async function main() {
     const client = new WebcrawlerClient(
         "YOUR API ACCESS KEY HERE",
     )
-    // sync way - promise will be resolved with the all the data
-    const syncJob = await client.crawl({
+
+    try {
+        const syncJob = await client.crawl({
             "items_limit": 10,
             "url": "https://stripe.com/",
             "scrape_type": "markdown"
-        }
-    )
-    for (const item of syncJob.job_items) {
-        item.getContent().then((content) => {
-            console.log(content.slice(0, 100));
         })
+        
+        for (const item of syncJob.job_items) {
+            const content = await item.getContent()
+            console.log(content.slice(0, 100))
+        }
+        console.log(syncJob)
+        
+    } catch (error) {
+        if (error instanceof webcrawlerapi.WebcrawlerApiError) {
+            console.error("Webcrawler error:", error.message)
+        }
+        return
     }
-    console.log(syncJob);
+}
+
+main().catch(console.error)
+```
+
+### Asynchronous Job
+In this mode, you get a job ID immediately and can poll for results. Asynchronous jobs are ideal for large-scale scraping operations where you want to process data in batches or handle long-running tasks efficiently.
 
-    // or async - get the job id and then poll the job status and get the data
+```javascript
+import webcrawlerapi from "webcrawlerapi-js"
 
-    const jobWithId = await client.crawlAsync({
+async function main() {
+    const client = new WebcrawlerClient(
+        "YOUR API ACCESS KEY HERE",
+    )
+
+    try {
+        // Get job ID immediately
+        const jobWithId = await client.crawlAsync({
             "items_limit": 10,
             "url": "https://stripe.com/",
             "scrape_type": "markdown"
+        })
+        
+        // Poll for job completion
+        const jobId = jobWithId.id
+        let asyncJob = await client.getJob(jobId)
+        
+        // Check job status
+        console.log("Job status:", asyncJob.status)
+        
+        // Once complete, access results
+        if (asyncJob.status === "completed") {
+            for (const item of asyncJob.job_items) {
+                const content = await item.getContent()
+                console.log(content.slice(0, 100))
+            }
         }
+    } catch (error) {
+        if (error instanceof webcrawlerapi.WebcrawlerApiError) {
+            console.error("Webcrawler error:", error.message)
+        }
+    }
+}
+
+main().catch(console.error)
+```
+
+## TypeScript Usage
+
+The SDK provides full TypeScript support with type safety and error handling. Our TypeScript integration ensures you catch potential issues at compile-time and provides excellent IDE support with autocompletion and inline documentation.
+
+```typescript
+import { WebcrawlerClient, WebcrawlerApiError } from "webcrawlerapi-js"
+
+async function main() {
+    const client = new WebcrawlerClient(
+        "YOUR_API_KEY",
+        // Optional base URL, defaults to production API
+        "http://localhost:8080" 
     )
-    // wait for job to complete
-    const jobId = jobWithId.id;
-    let asyncJob = await client.getJob(jobId);
 
-    console.log(asyncJob);
+    try {
+        // Basic crawl with markdown
+        const syncJob = await client.crawl({
+            "items_limit": 2,
+            "url": "https://books.toscrape.com/",
+            "scrape_type": "markdown"
+        })
+
+        // Type-safe status check
+        if (syncJob.status === "done") {
+            for (const item of syncJob.job_items) {
+                if (item.markdown_content_url) {
+                    // Access typed properties
+                    console.log("Status code:", item.page_status_code)
+                    console.log("Content URL:", item.markdown_content_url)
+                }
+            }
+        }
+
+        // Using specialized scraper
+        const scrapeJob = await client.scrape({
+            "crawler_id": "webcrawler/url-to-md",
+            "input": {
+                "url": "https://books.toscrape.com/"
+            }
+        })
+
+        if (scrapeJob.status === "done") {
+            const content = scrapeJob.getContent()
+            console.log("Scraped content:", content)
+        }
+
+    } catch (error) {
+        // Type-safe error handling
+        if (error instanceof WebcrawlerApiError) {
+            console.error("API Error:", error.errorCode, error.message)
+            // errorCode is typed and can be used for specific error handling
+            if (error.errorCode === "invalid_request") {
+                // Handle invalid request
+            }
+        }
+    }
 }
 
-main().catch(console.error);
+main().catch(console.error)
 ```
 
-## Response example
-```javascript
+### Error Handling
+
+The SDK provides typed error handling through the `WebcrawlerApiError` class. Our error handling system is designed to provide detailed, actionable feedback that helps you quickly identify and resolve issues in your scraping pipeline.
+
+```typescript
+try {
+    const syncJob = await client.crawl({
+        "items_limit": 2,
+        "url": "https://books.toscrape.com/",
+        // This will cause an error due to missing scrape_type
+    })
+} catch (error) {
+    if (error instanceof WebcrawlerApiError) {
+        // Typed error properties
+        console.error("Error code:", error.errorCode)
+        console.error("Error message:", error.message)
+    }
+}
+```
+
+## Response Structure
+
+Example of a job response. The response structure is carefully designed to provide comprehensive information about your scraping job, including detailed status updates, timing information, and resource usage metrics.
+
+```json
 {
   id: '49c4942b-b7d9-4d62-94b5-b54a3016ac51',
-  org_id: 'clxsnorta00075wuuqxgzzvxm',
   url: 'https://stripe.com/',
   scrape_type: 'markdown',
   whitelist_regexp: '',

package/dist/@types/api.d.ts

@@ -0,0 +1,15 @@
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse } from "./model";
+export declare class WebcrawlerClient {
+    protected apiVersion: string;
+    protected apiKey: string;
+    protected basePath: string;
+    constructor(apiKey: string, basePath?: string, apiVersion?: string);
+    scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
+    scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
+    scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
+    getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
+    crawl(crawlRequest: CrawlRequest): Promise<Job>;
+    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
+    getJob(jobID: string): Promise<Job>;
+    private sendRequest;
+}

package/dist/@types/constants.d.ts

@@ -0,0 +1,13 @@
+export declare const JobStatus: {
+    readonly IN_PROGRESS: "in_progress";
+    readonly NEW: "new";
+    readonly DONE: "done";
+};
+export declare const HttpStatus: {
+    readonly OK: 200;
+    readonly BAD_REQUEST: 400;
+    readonly UNAUTHORIZED: 401;
+    readonly FORBIDDEN: 403;
+    readonly NOT_FOUND: 404;
+    readonly INTERNAL_SERVER_ERROR: 500;
+};

package/dist/@types/errors.d.ts

@@ -0,0 +1,12 @@
+export declare class WebcrawlerApiError extends Error {
+    errorCode: string;
+    errorMessage: string;
+    statusCode?: number;
+    constructor(errorCode: string, errorMessage: string, statusCode?: number);
+}
+export interface ErrorResponse {
+    error_code?: string;
+    error_message?: string;
+    error?: string;
+}
+export declare function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError;

package/dist/@types/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./api";
+export * from "./model";
+export * from "./constants";
+export * from "./errors";

package/dist/@types/model.d.ts

@@ -0,0 +1,62 @@
+export interface ScrapeRequest {
+    input: any;
+    crawler_id: string;
+    max_retries?: number;
+    debug?: boolean;
+    omit_cache?: boolean;
+}
+export interface JobId {
+    id: string;
+}
+export interface ScrapeResponse {
+    id: string;
+    url: string;
+    status: any;
+    page_status_code: number;
+    created_at: string;
+    structured_data: any;
+    recommended_pull_delay_ms: number;
+}
+export interface CrawlRequest {
+    url: string;
+    items_limit?: number;
+    scrape_type?: string;
+    whitelist_regexp?: string;
+    blacklist_regexp?: string;
+    allow_subdomains?: boolean;
+    webhook_url?: string;
+}
+export interface Job {
+    id: string;
+    org_id: string;
+    url: string;
+    status: string;
+    scrape_type: string;
+    whitelist_regexp: string;
+    blacklist_regexp: string;
+    allow_subdomains: boolean;
+    items_limit: number;
+    created_at: string;
+    finished_at: string;
+    updated_at: string;
+    webhook_url: string;
+    recommended_pull_delay_ms: number;
+    job_items: JobItem[];
+}
+export interface JobItem {
+    id: string;
+    job_id: string;
+    original_url: string;
+    page_status_code: number;
+    markdown_content_url: string;
+    status: string;
+    title: string;
+    last_error: string;
+    created_at: string;
+    updated_at: string;
+    cost: number;
+    referred_url: string;
+    raw_content_url?: string;
+    cleaned_content_url?: string;
+    getContent(): Promise<string | null>;
+}

package/dist/api.js

@@ -1,16 +1,8 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WebcrawlerClient = void 0;
 const constants_1 = require("./constants");
+const errors_1 = require("./errors");
 const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
@@ -21,219 +13,183 @@ class WebcrawlerClient {
         this.basePath = basePath;
         this.apiVersion = apiVersion;
     }
-    scrapeAsync(scrapeRequest) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/scrape`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client"
-                },
-                'body': JSON.stringify(scrapeRequest),
-            };
-            const response = yield fetch(url, requestOptions);
-            if (response.ok) {
-                return response.json();
-            }
-            try {
-                const data = yield response.json();
-                throw new Error(`failed to scrape, response returned ${response.status} ${response.statusText}: ${data === null || data === void 0 ? void 0 : data.error}`);
-            }
-            catch (e) {
-                throw e;
-            }
-        });
+    async scrapeAsync(scrapeRequest) {
+        const url = `${this.basePath}/${this.apiVersion}/scrape`;
+        const requestOptions = {
+            'method': 'POST',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+            },
+            'body': JSON.stringify(scrapeRequest),
+        };
+        return await this.sendRequest(url, requestOptions);
     }
-    scrapeWithMeta(scrapeRequest_1) {
-        return __awaiter(this, arguments, void 0, function* (scrapeRequest, maxPollingRetries = MaxPullRetries) {
-            const url = `${this.basePath}/${this.apiVersion}/scrape`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`
-                },
-                'body': JSON.stringify(scrapeRequest),
-            };
-            const jobIdResponse = yield this.sendRequest(url, requestOptions);
-            if (jobIdResponse.id === '') {
-                throw new Error("Failed to fetch job status");
-            }
-            let delayIntervalMs = initialPullDelayMs;
-            for (let i = 0; i < maxPollingRetries; i++) {
-                yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-                const scrapeResult = yield this.getScrapeResult(jobIdResponse.id);
-                if (scrapeRequest.debug) {
-                    console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
-                }
-                if (scrapeResult.status !== constants_1.JobStatus.IN_PROGRESS && scrapeResult.status !== constants_1.JobStatus.NEW) {
-                    return scrapeResult;
-                }
-                if (scrapeResult.recommended_pull_delay_ms > 0) {
-                    delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
-                }
-            }
-            throw new Error("Scraping took too long, please retry or increase the number of polling retries");
-        });
+    async scrapeWithMeta(scrapeRequest, maxPollingRetries = MaxPullRetries) {
+        const url = `${this.basePath}/${this.apiVersion}/scrape`;
+        const requestOptions = {
+            'method': 'POST',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            },
+            'body': JSON.stringify(scrapeRequest),
+        };
+        const jobIdResponse = await this.sendRequest(url, requestOptions);
+        if (jobIdResponse.id === '') {
+            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
+        }
+        let delayIntervalMs = initialPullDelayMs;
+        for (let i = 0; i < maxPollingRetries; i++) {
+            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
+            const scrapeResult = await this.getScrapeResult(jobIdResponse.id);
+            if (scrapeRequest.debug) {
+                console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
+            }
+            if (scrapeResult.status !== constants_1.JobStatus.IN_PROGRESS && scrapeResult.status !== constants_1.JobStatus.NEW) {
+                return scrapeResult;
+            }
+            if (scrapeResult.recommended_pull_delay_ms > 0) {
+                delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
+            }
+        }
+        throw new errors_1.WebcrawlerApiError('timeout', 'Scraping took too long, please retry or increase the number of polling retries', 0);
     }
-    scrape(scrapeRequest_1) {
-        return __awaiter(this, arguments, void 0, function* (scrapeRequest, maxPollingRetries = MaxPullRetries) {
-            const scrapeResult = yield this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
-            return scrapeResult.structured_data;
-        });
+    async scrape(scrapeRequest, maxPollingRetries = MaxPullRetries) {
+        const scrapeResult = await this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
+        return scrapeResult.structured_data;
     }
-    getScrapeResult(scrapeID) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
-            const requestOptions = {
-                'method': 'GET',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                    'Cache-Control': 'no-cache, no-store, must-revalidate',
-                    'Pragma': 'no-cache',
-                    'Expires': '0'
-                },
-            };
-            const response = yield fetch(url, requestOptions);
-            if (response.ok) {
-                return response.json();
-            }
-            try {
-                const data = yield response.json();
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`);
-            }
-            catch (e) {
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}`);
-            }
-        });
+    async getScrapeResult(scrapeID) {
+        const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                'Cache-Control': 'no-cache, no-store, must-revalidate',
+                'Pragma': 'no-cache',
+                'Expires': '0'
+            },
+        };
+        return await this.sendRequest(url, requestOptions);
     }
-    crawl(crawlRequest) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/crawl`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                    'Cache-Control': 'no-cache, no-store, must-revalidate',
-                    'Pragma': 'no-cache',
-                    'Expires': '0'
-                },
-                'body': JSON.stringify(crawlRequest),
-            };
-            const jobIdResponse = yield this.sendRequest(url, requestOptions);
-            if (jobIdResponse.id === '') {
-                throw new Error("Failed to fetch job status");
-            }
-            let delayIntervalMs = initialPullDelayMs;
-            for (let i = 0; i < MaxPullRetries; i++) {
-                yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-                const timestamp = new Date().getTime();
-                const job = yield this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
-                if (job.status !== constants_1.JobStatus.IN_PROGRESS && job.status !== constants_1.JobStatus.NEW) {
-                    // Transform each job item to include getContent method
-                    job.job_items = job.job_items.map(item => (Object.assign(Object.assign({}, item), { getContent: function () {
-                            return __awaiter(this, void 0, void 0, function* () {
-                                if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
-                                    return null;
-                                }
-                                let contentUrl;
-                                switch (job.scrape_type) {
-                                    case 'html':
-                                        contentUrl = this.raw_content_url;
-                                        break;
-                                    case 'cleaned':
-                                        contentUrl = this.cleaned_content_url;
-                                        break;
-                                    case 'markdown':
-                                        contentUrl = this.markdown_content_url;
-                                        break;
-                                }
-                                if (!contentUrl) {
-                                    return null;
-                                }
-                                const response = yield fetch(contentUrl, {
-                                    headers: {
-                                        'Accept-Encoding': 'gzip, deflate, br',
-                                        'Accept': '*/*'
-                                    }
-                                });
-                                if (!response.ok) {
-                                    throw new Error(`Failed to fetch content: ${response.statusText}`);
-                                }
-                                return yield response.text();
-                            });
-                        } })));
-                    return job;
-                }
-                if (job.recommended_pull_delay_ms > 0) {
-                    delayIntervalMs = job.recommended_pull_delay_ms;
-                }
-            }
-            throw new Error("Crawling took too long, please retry or increase the number of polling retries");
-        });
+    async crawl(crawlRequest) {
+        const url = `${this.basePath}/${this.apiVersion}/crawl`;
+        const requestOptions = {
+            'method': 'POST',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                'Cache-Control': 'no-cache, no-store, must-revalidate',
+                'Pragma': 'no-cache',
+                'Expires': '0'
+            },
+            'body': JSON.stringify(crawlRequest),
+        };
+        const jobIdResponse = await this.sendRequest(url, requestOptions);
+        if (jobIdResponse.id === '') {
+            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
+        }
+        let delayIntervalMs = initialPullDelayMs;
+        for (let i = 0; i < MaxPullRetries; i++) {
+            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
+            const timestamp = new Date().getTime();
+            const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
+            if (job.status !== constants_1.JobStatus.IN_PROGRESS && job.status !== constants_1.JobStatus.NEW) {
+                // Transform each job item to include getContent method
+                job.job_items = job.job_items.map(item => ({
+                    ...item,
+                    getContent: async function () {
+                        if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
+                            return null;
+                        }
+                        let contentUrl;
+                        switch (job.scrape_type) {
+                            case 'html':
+                                contentUrl = this.raw_content_url;
+                                break;
+                            case 'cleaned':
+                                contentUrl = this.cleaned_content_url;
+                                break;
+                            case 'markdown':
+                                contentUrl = this.markdown_content_url;
+                                break;
+                        }
+                        if (!contentUrl) {
+                            return null;
+                        }
+                        const response = await fetch(contentUrl, {
+                            headers: {
+                                'Accept-Encoding': 'gzip, deflate, br',
+                                'Accept': '*/*'
+                            }
+                        });
+                        if (!response.ok) {
+                            throw new Error(`Failed to fetch content: ${response.statusText}`);
+                        }
+                        return await response.text();
+                    }
+                }));
+                return job;
+            }
+            if (job.recommended_pull_delay_ms > 0) {
+                delayIntervalMs = job.recommended_pull_delay_ms;
+            }
+        }
+        throw new errors_1.WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
     }
-    crawlAsync(crawlRequest) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/crawl`;
-            const requestOptions = {
-                'method': 'POST',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`
-                },
-                'body': JSON.stringify(crawlRequest),
-            };
-            return yield this.sendRequest(url, requestOptions);
-        });
+    async crawlAsync(crawlRequest) {
+        const url = `${this.basePath}/${this.apiVersion}/crawl`;
+        const requestOptions = {
+            'method': 'POST',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            },
+            'body': JSON.stringify(crawlRequest),
+        };
+        return await this.sendRequest(url, requestOptions);
     }
-    getJob(jobID) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
-            const requestOptions = {
-                'method': 'GET',
-                'headers': {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${this.apiKey}`,
-                    "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                    'Cache-Control': 'no-cache, no-store, must-revalidate',
-                    'Pragma': 'no-cache',
-                    'Expires': '0'
-                }
-            };
-            const response = yield fetch(url, requestOptions);
-            if (response.ok) {
-                return response.json();
-            }
-            try {
-                const data = yield response.json();
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`);
-            }
-            catch (e) {
-                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}`);
-            }
-        });
+    async getJob(jobID) {
+        const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                'Cache-Control': 'no-cache, no-store, must-revalidate',
+                'Pragma': 'no-cache',
+                'Expires': '0'
+            }
+        };
+        return await this.sendRequest(url, requestOptions);
     }
-    sendRequest(url, requestOptions) {
-        return __awaiter(this, void 0, void 0, function* () {
-            let response;
+    async sendRequest(url, requestOptions) {
+        let response;
+        try {
+            response = await fetch(url, requestOptions);
+        }
+        catch (e) {
+            throw new errors_1.WebcrawlerApiError('network_error', `Failed to send request: ${e}`, 0);
+        }
+        if (!response.ok) {
             try {
-                response = yield fetch(url, requestOptions);
+                const errorData = await response.json();
+                throw (0, errors_1.createErrorFromResponse)(response, errorData);
             }
             catch (e) {
-                throw new Error(`Failed to send request: ${e}`);
-            }
-            if (!response.ok) {
-                const errorResponse = yield response.json();
-                throw new Error(`${JSON.stringify(errorResponse)}`);
+                if (e instanceof errors_1.WebcrawlerApiError) {
+                    throw e;
+                }
+                // If we can't parse the error response, create a generic error
+                throw new errors_1.WebcrawlerApiError('unknown_error', `Request failed with status ${response.status} ${response.statusText}`, response.status);
             }
-            return response.json();
-        });
+        }
+        return response.json();
     }
 }
 exports.WebcrawlerClient = WebcrawlerClient;

package/dist/errors.d.ts

@@ -0,0 +1,12 @@
+export declare class WebcrawlerApiError extends Error {
+    errorCode: string;
+    errorMessage: string;
+    statusCode?: number;
+    constructor(errorCode: string, errorMessage: string, statusCode?: number);
+}
+export interface ErrorResponse {
+    error_code?: string;
+    error_message?: string;
+    error?: string;
+}
+export declare function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError;

package/dist/errors.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebcrawlerApiError = void 0;
+exports.createErrorFromResponse = createErrorFromResponse;
+class WebcrawlerApiError extends Error {
+    constructor(errorCode, errorMessage, statusCode) {
+        super(errorMessage);
+        this.name = 'WebcrawlerApiError';
+        this.errorCode = errorCode;
+        this.errorMessage = errorMessage;
+        this.statusCode = statusCode;
+    }
+}
+exports.WebcrawlerApiError = WebcrawlerApiError;
+function createErrorFromResponse(response, data) {
+    const errorCode = data.error_code || 'unknown_error';
+    const errorMessage = data.error_message || data.error || 'Unknown error';
+    return new WebcrawlerApiError(errorCode, errorMessage, response.status);
+}

package/dist/index.js

@@ -15,3 +15,6 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./api"), exports);
+__exportStar(require("./model"), exports);
+__exportStar(require("./constants"), exports);
+__exportStar(require("./errors"), exports);

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.0.9",
+  "version": "1.1.0",
 
-  "description": "JS client for WecrawlerAPI",
+  "description": "JS client for WebcrawlerAPI",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "scripts": {

package/{api.ts → src/api.ts}

@@ -1,5 +1,6 @@
 import {CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse} from "./model";
 import { JobStatus } from "./constants";
+import { WebcrawlerApiError, createErrorFromResponse, ErrorResponse } from "./errors";
 
 const BASE_PATH = "https://api.webcrawlerapi.com"
 const initialPullDelayMs = 2000
@@ -29,18 +30,7 @@ export class WebcrawlerClient {
             'body': JSON.stringify(scrapeRequest),
         };
 
-        const response = await fetch(url, requestOptions);
-        if (response.ok) {
-            return response.json();
-        }
-        try {
-            const data = await response.json();
-            throw new Error(
-                `failed to scrape, response returned ${response.status} ${response.statusText}: ${data?.error}`
-            );
-        } catch (e) {
-            throw e;
-        }
+        return await this.sendRequest(url, requestOptions);
     }
 
     public async scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<ScrapeResponse> {
@@ -58,7 +48,7 @@ export class WebcrawlerClient {
         const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
 
         if (jobIdResponse.id === '') {
-            throw new Error("Failed to fetch job status");
+            throw new WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
         }
 
         let delayIntervalMs = initialPullDelayMs;
@@ -75,7 +65,7 @@ export class WebcrawlerClient {
                 delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
             }
         }
-        throw new Error("Scraping took too long, please retry or increase the number of polling retries");
+        throw new WebcrawlerApiError('timeout', 'Scraping took too long, please retry or increase the number of polling retries', 0);
     }
 
     public async scrape(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<any> {
@@ -96,21 +86,7 @@ export class WebcrawlerClient {
                 'Expires': '0'
             },
         };
-        const response = await fetch(url, requestOptions);
-        if (response.ok) {
-            return response.json();
-        }
-
-        try {
-            const data = await response.json();
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`
-            );
-        } catch (e) {
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}`
-            );
-        }
+        return await this.sendRequest(url, requestOptions);
     }
 
     public async crawl(crawlRequest: CrawlRequest): Promise<Job> {
@@ -132,7 +108,7 @@ export class WebcrawlerClient {
         const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
 
         if (jobIdResponse.id === '') {
-            throw new Error("Failed to fetch job status");
+            throw new WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
         }
 
         let delayIntervalMs = initialPullDelayMs;
@@ -186,7 +162,7 @@ export class WebcrawlerClient {
                 delayIntervalMs = job.recommended_pull_delay_ms;
             }
         }
-        throw new Error("Crawling took too long, please retry or increase the number of polling retries");
+        throw new WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
     }
 
     public async crawlAsync(crawlRequest: CrawlRequest): Promise<JobId> {
@@ -216,36 +192,35 @@ export class WebcrawlerClient {
                 'Pragma': 'no-cache',
                 'Expires': '0'
             }
-        }
-        const response = await fetch(url, requestOptions);
-        if (response.ok) {
-            return response.json();
-        }
-
-        try {
-            const data = await response.json();
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`
-            );
-        } catch (e) {
-            throw new Error(
-                `failed to fetch job status ${response.status} ${response.statusText}`
-            );
-        }
+        };
+        return await this.sendRequest(url, requestOptions);
     }
 
-
     private async sendRequest(url: string, requestOptions: any): Promise<any> {
         let response: Response;
         try {
             response = await fetch(url, requestOptions);
         } catch (e) {
-            throw new Error(`Failed to send request: ${e}`);
+            throw new WebcrawlerApiError('network_error', `Failed to send request: ${e}`, 0);
         }
+
         if (!response.ok) {
-            const errorResponse = await response.json();
-            throw new Error(`${JSON.stringify(errorResponse)}`);
+            try {
+                const errorData = await response.json() as ErrorResponse;
+                throw createErrorFromResponse(response, errorData);
+            } catch (e) {
+                if (e instanceof WebcrawlerApiError) {
+                    throw e;
+                }
+                // If we can't parse the error response, create a generic error
+                throw new WebcrawlerApiError(
+                    'unknown_error',
+                    `Request failed with status ${response.status} ${response.statusText}`,
+                    response.status
+                );
+            }
         }
+
         return response.json();
     }
 }

package/src/errors.ts

@@ -0,0 +1,25 @@
+export class WebcrawlerApiError extends Error {
+    public errorCode: string;
+    public errorMessage: string;
+    public statusCode?: number;
+
+    constructor(errorCode: string, errorMessage: string, statusCode?: number) {
+        super(errorMessage);
+        this.name = 'WebcrawlerApiError';
+        this.errorCode = errorCode;
+        this.errorMessage = errorMessage;
+        this.statusCode = statusCode;
+    }
+}
+
+export interface ErrorResponse {
+    error_code?: string;
+    error_message?: string;
+    error?: string;
+}
+
+export function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError {
+    const errorCode = data.error_code || 'unknown_error';
+    const errorMessage = data.error_message || data.error || 'Unknown error';
+    return new WebcrawlerApiError(errorCode, errorMessage, response.status);
+} 

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from "./api";
+export * from "./model";
+export * from "./constants";
+export * from "./errors";

package/tsconfig.json

@@ -1,15 +1,21 @@
 {
   "compilerOptions": {
-    "declaration": true,
-    "target": "es6",
+    "lib": [
+      "es2021"
+    ],
     "module": "commonjs",
-    "noImplicitAny": true,
+    "target": "es2021",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
     "outDir": "dist",
-    "rootDir": ".",
-    "typeRoots": [
-      "node_modules/@types"
-    ]
+    "declarationDir": "dist/@types",
+    "declaration": true
   },
+  "include": [
+    "src/**/*"
+  ],
   "exclude": [
     "dist",
     "node_modules"

package/dist/index.d.ts

@@ -1 +0,0 @@
-export * from "./api";

package/index.ts

@@ -1 +0,0 @@
-export * from "./api";