webcrawlerapi-js 1.1.0 → 1.1.2

package/dist/@types/api.d.ts

@@ -1,4 +1,8 @@
 import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse } from "./model";
+import { JobStatus } from "./constants";
+import { WebcrawlerApiError } from "./errors";
+export { WebcrawlerApiError, JobStatus };
+export * from "./model";
 export declare class WebcrawlerClient {
     protected apiVersion: string;
     protected apiKey: string;

package/dist/api.js

@@ -1,11 +1,28 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.WebcrawlerClient = void 0;
+exports.WebcrawlerClient = exports.JobStatus = exports.WebcrawlerApiError = void 0;
 const constants_1 = require("./constants");
+Object.defineProperty(exports, "JobStatus", { enumerable: true, get: function () { return constants_1.JobStatus; } });
 const errors_1 = require("./errors");
+Object.defineProperty(exports, "WebcrawlerApiError", { enumerable: true, get: function () { return errors_1.WebcrawlerApiError; } });
 const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
+__exportStar(require("./model"), exports);
 class WebcrawlerClient {
     constructor(apiKey, basePath = BASE_PATH, apiVersion = "v1") {
         this.apiVersion = "v1";

package/package.json

@@ -1,10 +1,15 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.1.0",
+  "version": "1.1.2",
 
   "description": "JS client for WebcrawlerAPI",
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "main": "./dist/api.js",
+  "types": "./dist/@types/api.d.ts",
+  "files": [
+    "dist/",
+    "README.md",
+    "LICENSE"
+  ],
   "scripts": {
     "build": "tsc --outDir dist/"
   },

package/dist/@types/index.d.ts

@@ -1,4 +0,0 @@
-export * from "./api";
-export * from "./model";
-export * from "./constants";
-export * from "./errors";

package/dist/api.d.ts

@@ -1,15 +0,0 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse } from "./model";
-export declare class WebcrawlerClient {
-    protected apiVersion: string;
-    protected apiKey: string;
-    protected basePath: string;
-    constructor(apiKey: string, basePath?: string, apiVersion?: string);
-    scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
-    scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
-    scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
-    getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
-    crawl(crawlRequest: CrawlRequest): Promise<Job>;
-    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
-    getJob(jobID: string): Promise<Job>;
-    private sendRequest;
-}

package/dist/configuration.d.ts

@@ -1,16 +0,0 @@
-export interface ConfigurationParameters {
-    apiKey?: string;
-    basePath?: string;
-    apiVersion?: string;
-}
-export declare class Configuration {
-    /**
-     * Base options for http call
-     */
-    baseOptions?: any;
-    /**
-     * API version, for example: "v1"
-     */
-    apiVersion: string;
-    constructor(param?: ConfigurationParameters);
-}

package/dist/configuration.js

@@ -1,20 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Configuration = void 0;
-const packageJson = require("../package.json");
-class Configuration {
-    constructor(param = {}) {
-        /**
-         * API version, for example: "v1"
-         */
-        this.apiVersion = "v1";
-        this.apiVersion = param.apiVersion;
-        if (!this.baseOptions) {
-            this.baseOptions = {};
-        }
-        this.baseOptions = {
-            headers: Object.assign({ 'User-Agent': `webcrawlerapi-js/${packageJson.version}`, 'Content-Type': 'application/json' }, this.baseOptions.headers)
-        };
-    }
-}
-exports.Configuration = Configuration;

package/dist/constants.d.ts

@@ -1,13 +0,0 @@
-export declare const JobStatus: {
-    readonly IN_PROGRESS: "in_progress";
-    readonly NEW: "new";
-    readonly DONE: "done";
-};
-export declare const HttpStatus: {
-    readonly OK: 200;
-    readonly BAD_REQUEST: 400;
-    readonly UNAUTHORIZED: 401;
-    readonly FORBIDDEN: 403;
-    readonly NOT_FOUND: 404;
-    readonly INTERNAL_SERVER_ERROR: 500;
-};

package/dist/errors.d.ts

@@ -1,12 +0,0 @@
-export declare class WebcrawlerApiError extends Error {
-    errorCode: string;
-    errorMessage: string;
-    statusCode?: number;
-    constructor(errorCode: string, errorMessage: string, statusCode?: number);
-}
-export interface ErrorResponse {
-    error_code?: string;
-    error_message?: string;
-    error?: string;
-}
-export declare function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError;

package/dist/index.js

@@ -1,20 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./api"), exports);
-__exportStar(require("./model"), exports);
-__exportStar(require("./constants"), exports);
-__exportStar(require("./errors"), exports);

package/dist/model.d.ts

@@ -1,62 +0,0 @@
-export interface ScrapeRequest {
-    input: any;
-    crawler_id: string;
-    max_retries?: number;
-    debug?: boolean;
-    omit_cache?: boolean;
-}
-export interface JobId {
-    id: string;
-}
-export interface ScrapeResponse {
-    id: string;
-    url: string;
-    status: any;
-    page_status_code: number;
-    created_at: string;
-    structured_data: any;
-    recommended_pull_delay_ms: number;
-}
-export interface CrawlRequest {
-    url: string;
-    items_limit?: number;
-    scrape_type?: string;
-    whitelist_regexp?: string;
-    blacklist_regexp?: string;
-    allow_subdomains?: boolean;
-    webhook_url?: string;
-}
-export interface Job {
-    id: string;
-    org_id: string;
-    url: string;
-    status: string;
-    scrape_type: string;
-    whitelist_regexp: string;
-    blacklist_regexp: string;
-    allow_subdomains: boolean;
-    items_limit: number;
-    created_at: string;
-    finished_at: string;
-    updated_at: string;
-    webhook_url: string;
-    recommended_pull_delay_ms: number;
-    job_items: JobItem[];
-}
-export interface JobItem {
-    id: string;
-    job_id: string;
-    original_url: string;
-    page_status_code: number;
-    markdown_content_url: string;
-    status: string;
-    title: string;
-    last_error: string;
-    created_at: string;
-    updated_at: string;
-    cost: number;
-    referred_url: string;
-    raw_content_url?: string;
-    cleaned_content_url?: string;
-    getContent(): Promise<string | null>;
-}

package/src/api.ts

@@ -1,229 +0,0 @@
-import {CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse} from "./model";
-import { JobStatus } from "./constants";
-import { WebcrawlerApiError, createErrorFromResponse, ErrorResponse } from "./errors";
-
-const BASE_PATH = "https://api.webcrawlerapi.com"
-const initialPullDelayMs = 2000
-const MaxPullRetries = 100
-
-export class WebcrawlerClient {
-    protected apiVersion: string = "v1";
-    protected apiKey: string
-    protected basePath: string
-
-    constructor(apiKey: string, basePath: string = BASE_PATH, apiVersion: string = "v1") {
-        this.apiKey = apiKey ;
-        this.basePath = basePath;
-        this.apiVersion = apiVersion;
-    }
-
-    public async scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId> {
-        const url = `${this.basePath}/${this.apiVersion}/scrape`;
-
-        const requestOptions = {
-            'method': 'POST',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client"
-            },
-            'body': JSON.stringify(scrapeRequest),
-        };
-
-        return await this.sendRequest(url, requestOptions);
-    }
-
-    public async scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<ScrapeResponse> {
-        const url = `${this.basePath}/${this.apiVersion}/scrape`;
-
-        const requestOptions = {
-            'method': 'POST',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`
-            },
-            'body': JSON.stringify(scrapeRequest),
-        };
-
-        const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
-
-        if (jobIdResponse.id === '') {
-            throw new WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
-        }
-
-        let delayIntervalMs = initialPullDelayMs;
-        for (let i = 0; i < maxPollingRetries; i++) {
-            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-            const scrapeResult = await this.getScrapeResult(jobIdResponse.id);
-            if (scrapeRequest.debug) {
-                console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
-            }
-            if (scrapeResult.status !== JobStatus.IN_PROGRESS && scrapeResult.status !== JobStatus.NEW) {
-                return scrapeResult;
-            }
-            if (scrapeResult.recommended_pull_delay_ms > 0) {
-                delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
-            }
-        }
-        throw new WebcrawlerApiError('timeout', 'Scraping took too long, please retry or increase the number of polling retries', 0);
-    }
-
-    public async scrape(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<any> {
-        const scrapeResult = await this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
-        return scrapeResult.structured_data;
-    }
-
-    public async getScrapeResult(scrapeID: string): Promise<ScrapeResponse> {
-        const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
-        const requestOptions = {
-            'method': 'GET',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                'Cache-Control': 'no-cache, no-store, must-revalidate',
-                'Pragma': 'no-cache',
-                'Expires': '0'
-            },
-        };
-        return await this.sendRequest(url, requestOptions);
-    }
-
-    public async crawl(crawlRequest: CrawlRequest): Promise<Job> {
-        const url = `${this.basePath}/${this.apiVersion}/crawl`;
-
-        const requestOptions = {
-            'method': 'POST',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                'Cache-Control': 'no-cache, no-store, must-revalidate',
-                'Pragma': 'no-cache',
-                'Expires': '0'
-            },
-            'body': JSON.stringify(crawlRequest),
-        };
-
-        const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
-
-        if (jobIdResponse.id === '') {
-            throw new WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
-        }
-
-        let delayIntervalMs = initialPullDelayMs;
-        for (let i = 0; i < MaxPullRetries; i++) {
-            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
-            const timestamp = new Date().getTime();
-            const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
-            if (job.status !== JobStatus.IN_PROGRESS && job.status !== JobStatus.NEW) {
-                // Transform each job item to include getContent method
-                job.job_items = job.job_items.map(item => ({
-                    ...item,
-                    getContent: async function(): Promise<string | null> {
-                        if (job.status !== JobStatus.DONE || this.status !== JobStatus.DONE) {
-                            return null;
-                        }
-
-                        let contentUrl: string | undefined;
-                        switch (job.scrape_type) {
-                            case 'html':
-                                contentUrl = this.raw_content_url;
-                                break;
-                            case 'cleaned':
-                                contentUrl = this.cleaned_content_url;
-                                break;
-                            case 'markdown':
-                                contentUrl = this.markdown_content_url;
-                                break;
-                        }
-
-                        if (!contentUrl) {
-                            return null;
-                        }
-
-                        const response = await fetch(contentUrl, {
-                            headers: {
-                                'Accept-Encoding': 'gzip, deflate, br',
-                                'Accept': '*/*'
-                            }
-                        });
-
-                        if (!response.ok) {
-                            throw new Error(`Failed to fetch content: ${response.statusText}`);
-                        }
-
-                        return await response.text();
-                    }
-                }));
-                return job;
-            }
-            if (job.recommended_pull_delay_ms > 0) {
-                delayIntervalMs = job.recommended_pull_delay_ms;
-            }
-        }
-        throw new WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
-    }
-
-    public async crawlAsync(crawlRequest: CrawlRequest): Promise<JobId> {
-        const url = `${this.basePath}/${this.apiVersion}/crawl`;
-
-        const requestOptions = {
-            'method': 'POST',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`
-            },
-            'body': JSON.stringify(crawlRequest),
-        };
-
-        return await this.sendRequest(url, requestOptions);
-    }
-
-    public async getJob(jobID: string): Promise<Job> {
-        const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
-        const requestOptions = {
-            'method': 'GET',
-            'headers': {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`,
-                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
-                'Cache-Control': 'no-cache, no-store, must-revalidate',
-                'Pragma': 'no-cache',
-                'Expires': '0'
-            }
-        };
-        return await this.sendRequest(url, requestOptions);
-    }
-
-    private async sendRequest(url: string, requestOptions: any): Promise<any> {
-        let response: Response;
-        try {
-            response = await fetch(url, requestOptions);
-        } catch (e) {
-            throw new WebcrawlerApiError('network_error', `Failed to send request: ${e}`, 0);
-        }
-
-        if (!response.ok) {
-            try {
-                const errorData = await response.json() as ErrorResponse;
-                throw createErrorFromResponse(response, errorData);
-            } catch (e) {
-                if (e instanceof WebcrawlerApiError) {
-                    throw e;
-                }
-                // If we can't parse the error response, create a generic error
-                throw new WebcrawlerApiError(
-                    'unknown_error',
-                    `Request failed with status ${response.status} ${response.statusText}`,
-                    response.status
-                );
-            }
-        }
-
-        return response.json();
-    }
-}
-
-
-

package/src/constants.ts

@@ -1,14 +0,0 @@
-export const JobStatus = {
-    IN_PROGRESS: 'in_progress',
-    NEW: 'new',
-    DONE: 'done'
-} as const;
-
-export const HttpStatus = {
-    OK: 200,
-    BAD_REQUEST: 400,
-    UNAUTHORIZED: 401,
-    FORBIDDEN: 403,
-    NOT_FOUND: 404,
-    INTERNAL_SERVER_ERROR: 500
-} as const; 

package/src/errors.ts

@@ -1,25 +0,0 @@
-export class WebcrawlerApiError extends Error {
-    public errorCode: string;
-    public errorMessage: string;
-    public statusCode?: number;
-
-    constructor(errorCode: string, errorMessage: string, statusCode?: number) {
-        super(errorMessage);
-        this.name = 'WebcrawlerApiError';
-        this.errorCode = errorCode;
-        this.errorMessage = errorMessage;
-        this.statusCode = statusCode;
-    }
-}
-
-export interface ErrorResponse {
-    error_code?: string;
-    error_message?: string;
-    error?: string;
-}
-
-export function createErrorFromResponse(response: Response, data: ErrorResponse): WebcrawlerApiError {
-    const errorCode = data.error_code || 'unknown_error';
-    const errorMessage = data.error_message || data.error || 'Unknown error';
-    return new WebcrawlerApiError(errorCode, errorMessage, response.status);
-} 

package/src/index.ts

@@ -1,4 +0,0 @@
-export * from "./api";
-export * from "./model";
-export * from "./constants";
-export * from "./errors";

package/src/model.ts

@@ -1,67 +0,0 @@
-export interface ScrapeRequest {
-    input: any;
-    crawler_id: string;
-    max_retries?: number;
-    debug?: boolean;
-    omit_cache?: boolean;
-}
-
-export interface JobId {
-    id: string;
-}
-
-export interface ScrapeResponse {
-    id: string;
-    url: string;
-    status: any;
-    page_status_code: number;
-    created_at: string;
-    structured_data: any;
-    recommended_pull_delay_ms: number;
-}
-
-export interface CrawlRequest {
-    url: string;
-    items_limit?: number;
-    scrape_type?: string;
-    whitelist_regexp?: string;
-    blacklist_regexp?: string;
-    allow_subdomains?: boolean;
-    webhook_url?: string;
-}
-
-export interface Job {
-    id: string;
-    org_id: string;
-    url: string;
-    status: string;
-    scrape_type: string;
-    whitelist_regexp: string;
-    blacklist_regexp: string;
-    allow_subdomains: boolean;
-    items_limit: number;
-    created_at: string;
-    finished_at: string;
-    updated_at: string;
-    webhook_url: string;
-    recommended_pull_delay_ms: number;
-    job_items: JobItem[];
-}
-
-export interface JobItem {
-    id: string;
-    job_id: string;
-    original_url: string;
-    page_status_code: number;
-    markdown_content_url: string;
-    status: string;
-    title: string;
-    last_error: string;
-    created_at: string;
-    updated_at: string;
-    cost: number;
-    referred_url: string;
-    raw_content_url?: string;
-    cleaned_content_url?: string;
-    getContent(): Promise<string | null>;
-}

package/tsconfig.json

@@ -1,23 +0,0 @@
-{
-  "compilerOptions": {
-    "lib": [
-      "es2021"
-    ],
-    "module": "commonjs",
-    "target": "es2021",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "outDir": "dist",
-    "declarationDir": "dist/@types",
-    "declaration": true
-  },
-  "include": [
-    "src/**/*"
-  ],
-  "exclude": [
-    "dist",
-    "node_modules"
-  ]
-}