webcrawlerapi-js 1.1.1 → 1.1.3

package/dist/@types/api.d.ts

@@ -0,0 +1,19 @@
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse } from "./model";
+import { JobStatus } from "./constants";
+import { WebcrawlerApiError } from "./errors";
+export { WebcrawlerApiError, JobStatus };
+export * from "./model";
+export declare class WebcrawlerClient {
+    protected apiVersion: string;
+    protected apiKey: string;
+    protected basePath: string;
+    constructor(apiKey: string, basePath?: string, apiVersion?: string);
+    scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
+    scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
+    scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
+    getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
+    crawl(crawlRequest: CrawlRequest): Promise<Job>;
+    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
+    getJob(jobID: string): Promise<Job>;
+    private sendRequest;
+}

package/dist/@types/constants.d.ts

@@ -0,0 +1,13 @@
+export declare const JobStatus: {
+    readonly IN_PROGRESS: "in_progress";
+    readonly NEW: "new";
+    readonly DONE: "done";
+};
+export declare const HttpStatus: {
+    readonly OK: 200;
+    readonly BAD_REQUEST: 400;
+    readonly UNAUTHORIZED: 401;
+    readonly FORBIDDEN: 403;
+    readonly NOT_FOUND: 404;
+    readonly INTERNAL_SERVER_ERROR: 500;
+};

package/dist/@types/errors.d.ts

@@ -0,0 +1,12 @@
+export declare class WebcrawlerApiError extends Error {
+    errorCode: string;
+    errorMessage: string;
+    statusCode?: number;
+    constructor(errorCode: string, errorMessage: string, statusCode?: number);
+}
+export interface ErrorResponse {
+    error_code?: string;
+    error_message?: string;
+    error?: string;
+}
+export declare function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError;

package/{src/model.ts → dist/@types/model.d.ts}

@@ -5,11 +5,9 @@ export interface ScrapeRequest {
     debug?: boolean;
     omit_cache?: boolean;
 }
-
 export interface JobId {
     id: string;
 }
-
 export interface ScrapeResponse {
     id: string;
     url: string;
@@ -19,7 +17,6 @@ export interface ScrapeResponse {
     structured_data: any;
     recommended_pull_delay_ms: number;
 }
-
 export interface CrawlRequest {
     url: string;
     items_limit?: number;
@@ -29,7 +26,6 @@ export interface CrawlRequest {
     allow_subdomains?: boolean;
     webhook_url?: string;
 }
-
 export interface Job {
     id: string;
     org_id: string;
@@ -47,7 +43,6 @@ export interface Job {
     recommended_pull_delay_ms: number;
     job_items: JobItem[];
 }
-
 export interface JobItem {
     id: string;
     job_id: string;
@@ -64,4 +59,4 @@ export interface JobItem {
     raw_content_url?: string;
     cleaned_content_url?: string;
     getContent(): Promise<string | null>;
-}
+}

package/{src/api.ts → dist/api.js}

@@ -1,28 +1,37 @@
-import {CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse} from "./model";
-import { JobStatus } from "./constants";
-import { WebcrawlerApiError, createErrorFromResponse, ErrorResponse } from "./errors";
-
-const BASE_PATH = "https://api.webcrawlerapi.com"
-const initialPullDelayMs = 2000
-const MaxPullRetries = 100
-
-export { WebcrawlerApiError, JobStatus };
-export * from "./model";
-
-export class WebcrawlerClient {
-    protected apiVersion: string = "v1";
-    protected apiKey: string
-    protected basePath: string
-
-    constructor(apiKey: string, basePath: string = BASE_PATH, apiVersion: string = "v1") {
-        this.apiKey = apiKey ;
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebcrawlerClient = exports.JobStatus = exports.WebcrawlerApiError = void 0;
+const constants_1 = require("./constants");
+Object.defineProperty(exports, "JobStatus", { enumerable: true, get: function () { return constants_1.JobStatus; } });
+const errors_1 = require("./errors");
+Object.defineProperty(exports, "WebcrawlerApiError", { enumerable: true, get: function () { return errors_1.WebcrawlerApiError; } });
+const BASE_PATH = "https://api.webcrawlerapi.com";
+const initialPullDelayMs = 2000;
+const MaxPullRetries = 100;
+__exportStar(require("./model"), exports);
+class WebcrawlerClient {
+    constructor(apiKey, basePath = BASE_PATH, apiVersion = "v1") {
+        this.apiVersion = "v1";
+        this.apiKey = apiKey;
         this.basePath = basePath;
         this.apiVersion = apiVersion;
     }
-
-    public async scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId> {
+    async scrapeAsync(scrapeRequest) {
         const url = `${this.basePath}/${this.apiVersion}/scrape`;
-
         const requestOptions = {
             'method': 'POST',
             'headers': {
@@ -32,13 +41,10 @@ export class WebcrawlerClient {
             },
             'body': JSON.stringify(scrapeRequest),
         };
-
         return await this.sendRequest(url, requestOptions);
     }
-
-    public async scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<ScrapeResponse> {
+    async scrapeWithMeta(scrapeRequest, maxPollingRetries = MaxPullRetries) {
         const url = `${this.basePath}/${this.apiVersion}/scrape`;
-
         const requestOptions = {
             'method': 'POST',
             'headers': {
@@ -47,13 +53,10 @@ export class WebcrawlerClient {
             },
             'body': JSON.stringify(scrapeRequest),
         };
-
-        const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
-
+        const jobIdResponse = await this.sendRequest(url, requestOptions);
         if (jobIdResponse.id === '') {
-            throw new WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
+            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
         }
-
         let delayIntervalMs = initialPullDelayMs;
         for (let i = 0; i < maxPollingRetries; i++) {
             await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
@@ -61,22 +64,20 @@ export class WebcrawlerClient {
             if (scrapeRequest.debug) {
                 console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
             }
-            if (scrapeResult.status !== JobStatus.IN_PROGRESS && scrapeResult.status !== JobStatus.NEW) {
+            if (scrapeResult.status !== constants_1.JobStatus.IN_PROGRESS && scrapeResult.status !== constants_1.JobStatus.NEW) {
                 return scrapeResult;
             }
             if (scrapeResult.recommended_pull_delay_ms > 0) {
                 delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
             }
         }
-        throw new WebcrawlerApiError('timeout', 'Scraping took too long, please retry or increase the number of polling retries', 0);
+        throw new errors_1.WebcrawlerApiError('timeout', 'Scraping took too long, please retry or increase the number of polling retries', 0);
     }
-
-    public async scrape(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<any> {
+    async scrape(scrapeRequest, maxPollingRetries = MaxPullRetries) {
         const scrapeResult = await this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
         return scrapeResult.structured_data;
     }
-
-    public async getScrapeResult(scrapeID: string): Promise<ScrapeResponse> {
+    async getScrapeResult(scrapeID) {
         const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
         const requestOptions = {
             'method': 'GET',
@@ -91,10 +92,8 @@ export class WebcrawlerClient {
         };
         return await this.sendRequest(url, requestOptions);
     }
-
-    public async crawl(crawlRequest: CrawlRequest): Promise<Job> {
+    async crawl(crawlRequest) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
-
         const requestOptions = {
             'method': 'POST',
             'headers': {
@@ -107,28 +106,24 @@ export class WebcrawlerClient {
             },
             'body': JSON.stringify(crawlRequest),
         };
-
-        const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
-
+        const jobIdResponse = await this.sendRequest(url, requestOptions);
         if (jobIdResponse.id === '') {
-            throw new WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
+            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
         }
-
         let delayIntervalMs = initialPullDelayMs;
         for (let i = 0; i < MaxPullRetries; i++) {
             await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
             const timestamp = new Date().getTime();
             const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
-            if (job.status !== JobStatus.IN_PROGRESS && job.status !== JobStatus.NEW) {
+            if (job.status !== constants_1.JobStatus.IN_PROGRESS && job.status !== constants_1.JobStatus.NEW) {
                 // Transform each job item to include getContent method
                 job.job_items = job.job_items.map(item => ({
                     ...item,
-                    getContent: async function(): Promise<string | null> {
-                        if (job.status !== JobStatus.DONE || this.status !== JobStatus.DONE) {
+                    getContent: async function () {
+                        if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
                             return null;
                         }
-
-                        let contentUrl: string | undefined;
+                        let contentUrl;
                         switch (job.scrape_type) {
                             case 'html':
                                 contentUrl = this.raw_content_url;
@@ -140,22 +135,18 @@ export class WebcrawlerClient {
                                 contentUrl = this.markdown_content_url;
                                 break;
                         }
-
                         if (!contentUrl) {
                             return null;
                         }
-
                         const response = await fetch(contentUrl, {
                             headers: {
                                 'Accept-Encoding': 'gzip, deflate, br',
                                 'Accept': '*/*'
                             }
                         });
-
                         if (!response.ok) {
                             throw new Error(`Failed to fetch content: ${response.statusText}`);
                         }
-
                         return await response.text();
                     }
                 }));
@@ -165,12 +156,10 @@ export class WebcrawlerClient {
                 delayIntervalMs = job.recommended_pull_delay_ms;
             }
         }
-        throw new WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
+        throw new errors_1.WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
     }
-
-    public async crawlAsync(crawlRequest: CrawlRequest): Promise<JobId> {
+    async crawlAsync(crawlRequest) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
-
         const requestOptions = {
             'method': 'POST',
             'headers': {
@@ -179,11 +168,9 @@ export class WebcrawlerClient {
             },
             'body': JSON.stringify(crawlRequest),
         };
-
         return await this.sendRequest(url, requestOptions);
     }
-
-    public async getJob(jobID: string): Promise<Job> {
+    async getJob(jobID) {
         const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
         const requestOptions = {
             'method': 'GET',
@@ -198,35 +185,28 @@ export class WebcrawlerClient {
         };
         return await this.sendRequest(url, requestOptions);
     }
-
-    private async sendRequest(url: string, requestOptions: any): Promise<any> {
-        let response: Response;
+    async sendRequest(url, requestOptions) {
+        let response;
         try {
             response = await fetch(url, requestOptions);
-        } catch (e) {
-            throw new WebcrawlerApiError('network_error', `Failed to send request: ${e}`, 0);
         }
-
+        catch (e) {
+            throw new errors_1.WebcrawlerApiError('network_error', `Failed to send request: ${e}`, 0);
+        }
         if (!response.ok) {
             try {
-                const errorData = await response.json() as ErrorResponse;
-                throw createErrorFromResponse(response, errorData);
-            } catch (e) {
-                if (e instanceof WebcrawlerApiError) {
+                const errorData = await response.json();
+                throw (0, errors_1.createErrorFromResponse)(response, errorData);
+            }
+            catch (e) {
+                if (e instanceof errors_1.WebcrawlerApiError) {
                     throw e;
                 }
                 // If we can't parse the error response, create a generic error
-                throw new WebcrawlerApiError(
-                    'unknown_error',
-                    `Request failed with status ${response.status} ${response.statusText}`,
-                    response.status
-                );
+                throw new errors_1.WebcrawlerApiError('unknown_error', `Request failed with status ${response.status} ${response.statusText}`, response.status);
             }
         }
-
         return response.json();
     }
 }
-
-
-
+exports.WebcrawlerClient = WebcrawlerClient;

package/{src/constants.ts → dist/constants.js}

@@ -1,14 +1,16 @@
-export const JobStatus = {
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpStatus = exports.JobStatus = void 0;
+exports.JobStatus = {
     IN_PROGRESS: 'in_progress',
     NEW: 'new',
     DONE: 'done'
-} as const;
-
-export const HttpStatus = {
+};
+exports.HttpStatus = {
     OK: 200,
     BAD_REQUEST: 400,
     UNAUTHORIZED: 401,
     FORBIDDEN: 403,
     NOT_FOUND: 404,
     INTERNAL_SERVER_ERROR: 500
-} as const; 
+};

package/dist/errors.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebcrawlerApiError = void 0;
+exports.createErrorFromResponse = createErrorFromResponse;
+class WebcrawlerApiError extends Error {
+    constructor(errorCode, errorMessage, statusCode) {
+        super(errorMessage);
+        this.name = 'WebcrawlerApiError';
+        this.errorCode = errorCode;
+        this.errorMessage = errorMessage;
+        this.statusCode = statusCode;
+    }
+}
+exports.WebcrawlerApiError = WebcrawlerApiError;
+function createErrorFromResponse(response, data) {
+    const errorCode = data.error_code || 'unknown_error';
+    const errorMessage = data.error_message || data.error || 'Unknown error';
+    return new WebcrawlerApiError(errorCode, errorMessage, response.status);
+}

package/dist/model.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,10 +1,15 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.1.1",
+  "version": "1.1.3",
 
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",
+  "files": [
+    "dist/",
+    "README.md",
+    "LICENSE"
+  ],
   "scripts": {
     "build": "tsc --outDir dist/"
   },
@@ -21,7 +26,7 @@
     "@types/node": "^22.4.0",
     "typescript": "^5.5.4"
   },
-  "author": "Andrew <sdk@webcrawlerapi.com>",
+  "author": "WebcrawlerAPI <sdk@webcrawlerapi.com>",
   "license": "MIT",
   "dependencies": {}
 }

package/src/errors.ts

@@ -1,25 +0,0 @@
-export class WebcrawlerApiError extends Error {
-    public errorCode: string;
-    public errorMessage: string;
-    public statusCode?: number;
-
-    constructor(errorCode: string, errorMessage: string, statusCode?: number) {
-        super(errorMessage);
-        this.name = 'WebcrawlerApiError';
-        this.errorCode = errorCode;
-        this.errorMessage = errorMessage;
-        this.statusCode = statusCode;
-    }
-}
-
-export interface ErrorResponse {
-    error_code?: string;
-    error_message?: string;
-    error?: string;
-}
-
-export function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError {
-    const errorCode = data.error_code || 'unknown_error';
-    const errorMessage = data.error_message || data.error || 'Unknown error';
-    return new WebcrawlerApiError(errorCode, errorMessage, response.status);
-} 

package/tsconfig.json

@@ -1,23 +0,0 @@
-{
-  "compilerOptions": {
-    "lib": [
-      "es2021"
-    ],
-    "module": "commonjs",
-    "target": "es2021",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "outDir": "dist",
-    "declarationDir": "dist/@types",
-    "declaration": true
-  },
-  "include": [
-    "src/**/*"
-  ],
-  "exclude": [
-    "dist",
-    "node_modules"
-  ]
-}