webcrawlerapi-js 1.1.0 → 1.1.1

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.1.0",
+  "version": "1.1.1",
 
   "description": "JS client for WebcrawlerAPI",
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "main": "./dist/api.js",
+  "types": "./dist/@types/api.d.ts",
   "scripts": {
     "build": "tsc --outDir dist/"
   },

package/src/api.ts

@@ -6,6 +6,9 @@ const BASE_PATH = "https://api.webcrawlerapi.com"
 const initialPullDelayMs = 2000
 const MaxPullRetries = 100
 
+export { WebcrawlerApiError, JobStatus };
+export * from "./model";
+
 export class WebcrawlerClient {
     protected apiVersion: string = "v1";
     protected apiKey: string

package/dist/@types/api.d.ts

@@ -1,15 +0,0 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse } from "./model";
-export declare class WebcrawlerClient {
-    protected apiVersion: string;
-    protected apiKey: string;
-    protected basePath: string;
-    constructor(apiKey: string, basePath?: string, apiVersion?: string);
-    scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
-    scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
-    scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
-    getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
-    crawl(crawlRequest: CrawlRequest): Promise<Job>;
-    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
-    getJob(jobID: string): Promise<Job>;
-    private sendRequest;
-}

package/dist/@types/constants.d.ts

@@ -1,13 +0,0 @@
-export declare const JobStatus: {
-    readonly IN_PROGRESS: "in_progress";
-    readonly NEW: "new";
-    readonly DONE: "done";
-};
-export declare const HttpStatus: {
-    readonly OK: 200;
-    readonly BAD_REQUEST: 400;
-    readonly UNAUTHORIZED: 401;
-    readonly FORBIDDEN: 403;
-    readonly NOT_FOUND: 404;
-    readonly INTERNAL_SERVER_ERROR: 500;
-};

package/dist/@types/errors.d.ts

@@ -1,12 +0,0 @@
-export declare class WebcrawlerApiError extends Error {
-    errorCode: string;
-    errorMessage: string;
-    statusCode?: number;
-    constructor(errorCode: string, errorMessage: string, statusCode?: number);
-}
-export interface ErrorResponse {
-    error_code?: string;
-    error_message?: string;
-    error?: string;
-}
-export declare function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError;

package/dist/@types/index.d.ts

@@ -1,4 +0,0 @@
-export * from "./api";
-export * from "./model";
-export * from "./constants";
-export * from "./errors";

package/dist/@types/model.d.ts

@@ -1,62 +0,0 @@
-export interface ScrapeRequest {
-    input: any;
-    crawler_id: string;
-    max_retries?: number;
-    debug?: boolean;
-    omit_cache?: boolean;
-}
-export interface JobId {
-    id: string;
-}
-export interface ScrapeResponse {
-    id: string;
-    url: string;
-    status: any;
-    page_status_code: number;
-    created_at: string;
-    structured_data: any;
-    recommended_pull_delay_ms: number;
-}
-export interface CrawlRequest {
-    url: string;
-    items_limit?: number;
-    scrape_type?: string;
-    whitelist_regexp?: string;
-    blacklist_regexp?: string;
-    allow_subdomains?: boolean;
-    webhook_url?: string;
-}
-export interface Job {
-    id: string;
-    org_id: string;
-    url: string;
-    status: string;
-    scrape_type: string;
-    whitelist_regexp: string;
-    blacklist_regexp: string;
-    allow_subdomains: boolean;
-    items_limit: number;
-    created_at: string;
-    finished_at: string;
-    updated_at: string;
-    webhook_url: string;
-    recommended_pull_delay_ms: number;
-    job_items: JobItem[];
-}
-export interface JobItem {
-    id: string;
-    job_id: string;
-    original_url: string;
-    page_status_code: number;
-    markdown_content_url: string;
-    status: string;
-    title: string;
-    last_error: string;
-    created_at: string;
-    updated_at: string;
-    cost: number;
-    referred_url: string;
-    raw_content_url?: string;
-    cleaned_content_url?: string;
-    getContent(): Promise<string | null>;
-}

package/dist/api.d.ts

@@ -1,15 +0,0 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse } from "./model";
-export declare class WebcrawlerClient {
-    protected apiVersion: string;
-    protected apiKey: string;
-    protected basePath: string;
-    constructor(apiKey: string, basePath?: string, apiVersion?: string);
-    scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
-    scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
-    scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
-    getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
-    crawl(crawlRequest: CrawlRequest): Promise<Job>;
-    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
-    getJob(jobID: string): Promise<Job>;
-    private sendRequest;
-}

package/dist/api.js

@@ -1,195 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WebcrawlerClient = void 0;
-const constants_1 = require("./constants");
-const errors_1 = require("./errors");
-const BASE_PATH = "https://api.webcrawlerapi.com";
-const initialPullDelayMs = 2000;
-const MaxPullRetries = 100;
-class WebcrawlerClient {
-    constructor(apiKey, basePath = BASE_PATH, apiVersion = "v1") {
-        this.apiVersion = "v1";
-        this.apiKey = apiKey;
-        this.basePath = basePath;
-        this.apiVersion = apiVersion;
-    }
-    async scrapeAsync(scrapeRequest) {
-        const url = `${this.basePath}/${this.apiVersion}/scrape`;
-        const requestOptions = {
-            'method': 'POST',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
-            },
-            'body': JSON.stringify(scrapeRequest),
-        };
-        return await this.sendRequest(url, requestOptions);
-    }
-    async scrapeWithMeta(scrapeRequest, maxPollingRetries = MaxPullRetries) {
-        const url = `${this.basePath}/${this.apiVersion}/scrape`;
-        const requestOptions = {
-            'method': 'POST',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`
-            },
-            'body': JSON.stringify(scrapeRequest),
-        };
-        const jobIdResponse = await this.sendRequest(url, requestOptions);
-        if (jobIdResponse.id === '') {
-            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
-        }
-        let delayIntervalMs = initialPullDelayMs;
-        for (let i = 0; i < maxPollingRetries; i++) {
-            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-            const scrapeResult = await this.getScrapeResult(jobIdResponse.id);
-            if (scrapeRequest.debug) {
-                console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
-            }
-            if (scrapeResult.status !== constants_1.JobStatus.IN_PROGRESS && scrapeResult.status !== constants_1.JobStatus.NEW) {
-                return scrapeResult;
-            }
-            if (scrapeResult.recommended_pull_delay_ms > 0) {
-                delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
-            }
-        }
-        throw new errors_1.WebcrawlerApiError('timeout', 'Scraping took too long, please retry or increase the number of polling retries', 0);
-    }
-    async scrape(scrapeRequest, maxPollingRetries = MaxPullRetries) {
-        const scrapeResult = await this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
-        return scrapeResult.structured_data;
-    }
-    async getScrapeResult(scrapeID) {
-        const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
-        const requestOptions = {
-            'method': 'GET',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                'Cache-Control': 'no-cache, no-store, must-revalidate',
-                'Pragma': 'no-cache',
-                'Expires': '0'
-            },
-        };
-        return await this.sendRequest(url, requestOptions);
-    }
-    async crawl(crawlRequest) {
-        const url = `${this.basePath}/${this.apiVersion}/crawl`;
-        const requestOptions = {
-            'method': 'POST',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                'Cache-Control': 'no-cache, no-store, must-revalidate',
-                'Pragma': 'no-cache',
-                'Expires': '0'
-            },
-            'body': JSON.stringify(crawlRequest),
-        };
-        const jobIdResponse = await this.sendRequest(url, requestOptions);
-        if (jobIdResponse.id === '') {
-            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
-        }
-        let delayIntervalMs = initialPullDelayMs;
-        for (let i = 0; i < MaxPullRetries; i++) {
-            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-            const timestamp = new Date().getTime();
-            const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
-            if (job.status !== constants_1.JobStatus.IN_PROGRESS && job.status !== constants_1.JobStatus.NEW) {
-                // Transform each job item to include getContent method
-                job.job_items = job.job_items.map(item => ({
-                    ...item,
-                    getContent: async function () {
-                        if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
-                            return null;
-                        }
-                        let contentUrl;
-                        switch (job.scrape_type) {
-                            case 'html':
-                                contentUrl = this.raw_content_url;
-                                break;
-                            case 'cleaned':
-                                contentUrl = this.cleaned_content_url;
-                                break;
-                            case 'markdown':
-                                contentUrl = this.markdown_content_url;
-                                break;
-                        }
-                        if (!contentUrl) {
-                            return null;
-                        }
-                        const response = await fetch(contentUrl, {
-                            headers: {
-                                'Accept-Encoding': 'gzip, deflate, br',
-                                'Accept': '*/*'
-                            }
-                        });
-                        if (!response.ok) {
-                            throw new Error(`Failed to fetch content: ${response.statusText}`);
-                        }
-                        return await response.text();
-                    }
-                }));
-                return job;
-            }
-            if (job.recommended_pull_delay_ms > 0) {
-                delayIntervalMs = job.recommended_pull_delay_ms;
-            }
-        }
-        throw new errors_1.WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
-    }
-    async crawlAsync(crawlRequest) {
-        const url = `${this.basePath}/${this.apiVersion}/crawl`;
-        const requestOptions = {
-            'method': 'POST',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`
-            },
-            'body': JSON.stringify(crawlRequest),
-        };
-        return await this.sendRequest(url, requestOptions);
-    }
-    async getJob(jobID) {
-        const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
-        const requestOptions = {
-            'method': 'GET',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                'Cache-Control': 'no-cache, no-store, must-revalidate',
-                'Pragma': 'no-cache',
-                'Expires': '0'
-            }
-        };
-        return await this.sendRequest(url, requestOptions);
-    }
-    async sendRequest(url, requestOptions) {
-        let response;
-        try {
-            response = await fetch(url, requestOptions);
-        }
-        catch (e) {
-            throw new errors_1.WebcrawlerApiError('network_error', `Failed to send request: ${e}`, 0);
-        }
-        if (!response.ok) {
-            try {
-                const errorData = await response.json();
-                throw (0, errors_1.createErrorFromResponse)(response, errorData);
-            }
-            catch (e) {
-                if (e instanceof errors_1.WebcrawlerApiError) {
-                    throw e;
-                }
-                // If we can't parse the error response, create a generic error
-                throw new errors_1.WebcrawlerApiError('unknown_error', `Request failed with status ${response.status} ${response.statusText}`, response.status);
-            }
-        }
-        return response.json();
-    }
-}
-exports.WebcrawlerClient = WebcrawlerClient;

package/dist/configuration.d.ts

@@ -1,16 +0,0 @@
-export interface ConfigurationParameters {
-    apiKey?: string;
-    basePath?: string;
-    apiVersion?: string;
-}
-export declare class Configuration {
-    /**
-     * Base options for http call
-     */
-    baseOptions?: any;
-    /**
-     * API version, for example: "v1"
-     */
-    apiVersion: string;
-    constructor(param?: ConfigurationParameters);
-}

package/dist/configuration.js

@@ -1,20 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Configuration = void 0;
-const packageJson = require("../package.json");
-class Configuration {
-    constructor(param = {}) {
-        /**
-         * API version, for example: "v1"
-         */
-        this.apiVersion = "v1";
-        this.apiVersion = param.apiVersion;
-        if (!this.baseOptions) {
-            this.baseOptions = {};
-        }
-        this.baseOptions = {
-            headers: Object.assign({ 'User-Agent': `webcrawlerapi-js/${packageJson.version}`, 'Content-Type': 'application/json' }, this.baseOptions.headers)
-        };
-    }
-}
-exports.Configuration = Configuration;

package/dist/constants.d.ts

@@ -1,13 +0,0 @@
-export declare const JobStatus: {
-    readonly IN_PROGRESS: "in_progress";
-    readonly NEW: "new";
-    readonly DONE: "done";
-};
-export declare const HttpStatus: {
-    readonly OK: 200;
-    readonly BAD_REQUEST: 400;
-    readonly UNAUTHORIZED: 401;
-    readonly FORBIDDEN: 403;
-    readonly NOT_FOUND: 404;
-    readonly INTERNAL_SERVER_ERROR: 500;
-};

package/dist/constants.js

@@ -1,16 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.HttpStatus = exports.JobStatus = void 0;
-exports.JobStatus = {
-    IN_PROGRESS: 'in_progress',
-    NEW: 'new',
-    DONE: 'done'
-};
-exports.HttpStatus = {
-    OK: 200,
-    BAD_REQUEST: 400,
-    UNAUTHORIZED: 401,
-    FORBIDDEN: 403,
-    NOT_FOUND: 404,
-    INTERNAL_SERVER_ERROR: 500
-};

package/dist/errors.d.ts

@@ -1,12 +0,0 @@
-export declare class WebcrawlerApiError extends Error {
-    errorCode: string;
-    errorMessage: string;
-    statusCode?: number;
-    constructor(errorCode: string, errorMessage: string, statusCode?: number);
-}
-export interface ErrorResponse {
-    error_code?: string;
-    error_message?: string;
-    error?: string;
-}
-export declare function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError;

package/dist/errors.js

@@ -1,19 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WebcrawlerApiError = void 0;
-exports.createErrorFromResponse = createErrorFromResponse;
-class WebcrawlerApiError extends Error {
-    constructor(errorCode, errorMessage, statusCode) {
-        super(errorMessage);
-        this.name = 'WebcrawlerApiError';
-        this.errorCode = errorCode;
-        this.errorMessage = errorMessage;
-        this.statusCode = statusCode;
-    }
-}
-exports.WebcrawlerApiError = WebcrawlerApiError;
-function createErrorFromResponse(response, data) {
-    const errorCode = data.error_code || 'unknown_error';
-    const errorMessage = data.error_message || data.error || 'Unknown error';
-    return new WebcrawlerApiError(errorCode, errorMessage, response.status);
-}

package/dist/index.js

@@ -1,20 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./api"), exports);
-__exportStar(require("./model"), exports);
-__exportStar(require("./constants"), exports);
-__exportStar(require("./errors"), exports);

package/dist/model.d.ts

@@ -1,62 +0,0 @@
-export interface ScrapeRequest {
-    input: any;
-    crawler_id: string;
-    max_retries?: number;
-    debug?: boolean;
-    omit_cache?: boolean;
-}
-export interface JobId {
-    id: string;
-}
-export interface ScrapeResponse {
-    id: string;
-    url: string;
-    status: any;
-    page_status_code: number;
-    created_at: string;
-    structured_data: any;
-    recommended_pull_delay_ms: number;
-}
-export interface CrawlRequest {
-    url: string;
-    items_limit?: number;
-    scrape_type?: string;
-    whitelist_regexp?: string;
-    blacklist_regexp?: string;
-    allow_subdomains?: boolean;
-    webhook_url?: string;
-}
-export interface Job {
-    id: string;
-    org_id: string;
-    url: string;
-    status: string;
-    scrape_type: string;
-    whitelist_regexp: string;
-    blacklist_regexp: string;
-    allow_subdomains: boolean;
-    items_limit: number;
-    created_at: string;
-    finished_at: string;
-    updated_at: string;
-    webhook_url: string;
-    recommended_pull_delay_ms: number;
-    job_items: JobItem[];
-}
-export interface JobItem {
-    id: string;
-    job_id: string;
-    original_url: string;
-    page_status_code: number;
-    markdown_content_url: string;
-    status: string;
-    title: string;
-    last_error: string;
-    created_at: string;
-    updated_at: string;
-    cost: number;
-    referred_url: string;
-    raw_content_url?: string;
-    cleaned_content_url?: string;
-    getContent(): Promise<string | null>;
-}

package/dist/model.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/src/index.ts

@@ -1,4 +0,0 @@
-export * from "./api";
-export * from "./model";
-export * from "./constants";
-export * from "./errors";