webcrawlerapi-js 1.0.4

package/README.md

@@ -0,0 +1,60 @@
+# JS client for WebcrawlerAPI scrapers
+
+Official client for [WebcrawlerAPI](https://webcrawlerapi.com/) scrapers.
+
+## Installation
+Install WebcrawlerAPI js package:
+
+`npm i webcrawlerapi-js`
+
+## Preparation
+1. Register to [dashboard|https://dash.webcrawlerapi.com/].
+2. Get an [Access Key](https://dash.webcrawlerapi.com/access).
+
+## Request example
+
+```javascript
+const webcrawlerapi = require('webcrawlerapi-js');
+
+async function main() {
+    const client = new webcrawlerapi.WebcrawlerClient(
+        "YOUR API KEY HERE"
+    )
+    const response = await client.scrape({
+        input: {
+            "url": "https://www.funda.nl/detail/koop/heerhugowaard/huis-govert-flinckplantsoen-1/89968455/"
+        },
+        crawler_id: "webcrawler/funda",
+    })
+    console.log(response)
+}
+
+main().catch(console.error);
+```
+
+## Response example
+```json
+{
+  city: 'Heerhugowaard',
+  price: 325000,
+  images: [ 'https://cloud.funda.nl/valentina_media/191/215/183_2160.jpg' ],
+  status: 'inonderhandeling',
+  videos: [],
+  address: 'Govert Flinckplantsoen 1',
+  country: 'Nederland',
+  province: 'Noord-Holland',
+  plot_area: '183 m²',
+  post_code: '1701NH',
+  description: 'De woning is met liefde en zorg 53 jaar bewoond door...',
+  living_area: 127,
+  house_number: 1,
+  energie_label: 'd',
+  property_type: 'woonhuis',
+  publication_date: '2024-05-28T00:00:00',
+  number_of_bedrooms: 4,
+  coordinates_latitude: 52.67685,
+  year_of_construction: 1971,
+  coordinates_longitude: 4.8560443,
+  house_number_extension: ''
+}
+```

package/api.ts

@@ -0,0 +1,198 @@
+import {CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse} from "./model";
+
+const BASE_PATH = "https://api.webcrawlerapi.com"
+const initialPullDelayMs = 2000
+const MaxPullRetries = 100
+
+export class WebcrawlerClient {
+    protected apiVersion: string = "v1";
+    protected apiKey: string
+    protected basePath: string
+
+    constructor(apiKey: string, basePath: string = BASE_PATH, apiVersion: string = "v1") {
+        this.apiKey = apiKey ;
+        this.basePath = basePath;
+        this.apiVersion = apiVersion;
+    }
+
+    public async scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId> {
+        const url = `${this.basePath}/${this.apiVersion}/scrape`;
+
+        const requestOptions = {
+            'method': 'POST',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            },
+            'body': JSON.stringify(scrapeRequest),
+        };
+
+        const response = await fetch(url, requestOptions);
+        if (response.ok) {
+            return response.json();
+        }
+        try {
+            const data = await response.json();
+            throw new Error(
+                `failed to scrape, response returned ${response.status} ${response.statusText}: ${data?.error}`
+            );
+        } catch (e) {
+            throw e;
+        }
+    }
+
+    public async scrapeWithMeta(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse> {
+        const url = `${this.basePath}/${this.apiVersion}/scrape`;
+
+        const requestOptions = {
+            'method': 'POST',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            },
+            'body': JSON.stringify(scrapeRequest),
+        };
+
+        const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
+
+        if (jobIdResponse.id === '') {
+            throw new Error("Failed to fetch job status");
+        }
+
+        let delayIntervalMs = initialPullDelayMs;
+        for (let i = 0; i < MaxPullRetries; i++) {
+            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
+            const scrapeResult = await this.getScrapeResult(jobIdResponse.id);
+            if (scrapeRequest.debug) {
+                console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
+            }
+            if (scrapeResult.status !== 'in_progress' && scrapeResult.status !== 'new') {
+                return scrapeResult;
+            }
+            if (scrapeResult.recommended_pull_delay_ms > 0) {
+                delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
+            }
+        }
+        throw new Error("Scraping took too long, please retry");
+    }
+
+    public async scrape(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse> {
+        const scrapeResult = await this.scrapeWithMeta(scrapeRequest);
+        return scrapeResult.structured_data;
+    }
+
+    public async getScrapeResult(scrapeID: string): Promise<ScrapeResponse> {
+        const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': {
+                'Authorization': `Bearer ${this.apiKey}`,
+            },
+        };
+        const response = await fetch(url, requestOptions);
+        if (response.ok) {
+            return response.json();
+        }
+
+        try {
+            const data = await response.json();
+            throw new Error(
+                `failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`
+            );
+        } catch (e) {
+            throw new Error(
+                `failed to fetch job status ${response.status} ${response.statusText}`
+            );
+        }
+    }
+
+    public async crawl(crawlRequest: CrawlRequest): Promise<Job> {
+        const url = `${this.basePath}/${this.apiVersion}/crawl`;
+
+        const requestOptions = {
+            'method': 'POST',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            },
+            'body': JSON.stringify(crawlRequest),
+        };
+
+        const jobIdResponse: JobId = await this.sendRequest(url, requestOptions);
+
+        if (jobIdResponse.id === '') {
+            throw new Error("Failed to fetch job status");
+        }
+
+        let delayIntervalMs = initialPullDelayMs;
+        for (let i = 0; i < MaxPullRetries; i++) {
+            await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
+            const job = await this.getJob(jobIdResponse.id);
+            if (job.status !== 'in_progress' && job.status !== 'new') {
+                return job;
+            }
+            if (job.recommended_pull_delay_ms > 0) {
+                delayIntervalMs = job.recommended_pull_delay_ms;
+            }
+        }
+    }
+
+    public async crawlAsync(crawlRequest: CrawlRequest): Promise<JobId> {
+        const url = `${this.basePath}/${this.apiVersion}/crawl`;
+
+        const requestOptions = {
+            'method': 'POST',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            },
+            'body': JSON.stringify(crawlRequest),
+        };
+
+        return await this.sendRequest(url, requestOptions);
+    }
+
+    public async getJob(jobID: string): Promise<Job> {
+        const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            }
+        }
+        const response = await fetch(url, requestOptions);
+        if (response.ok) {
+            return response.json();
+        }
+
+        try {
+            const data = await response.json();
+            throw new Error(
+                `failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`
+            );
+        } catch (e) {
+            throw new Error(
+                `failed to fetch job status ${response.status} ${response.statusText}`
+            );
+        }
+    }
+
+
+    private async sendRequest(url: string, requestOptions: any): Promise<any> {
+        let response: Response;
+        try {
+            response = await fetch(url, requestOptions);
+        } catch (e) {
+            throw new Error(`Failed to send request: ${e}`);
+        }
+        if (!response.ok) {
+            const errorResponse = await response.json();
+            throw new Error(`${JSON.stringify(errorResponse)}`);
+        }
+        return response.json();
+    }
+}
+
+
+

package/dist/api.d.ts

@@ -0,0 +1,15 @@
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse } from "./model";
+export declare class WebcrawlerClient {
+    protected apiVersion: string;
+    protected apiKey: string;
+    protected basePath: string;
+    constructor(apiKey: string, basePath?: string, apiVersion?: string);
+    scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
+    scrapeWithMeta(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse>;
+    scrape(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse>;
+    getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
+    crawl(crawlRequest: CrawlRequest): Promise<Job>;
+    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
+    getJob(jobID: string): Promise<Job>;
+    private sendRequest;
+}

package/dist/api.js

@@ -0,0 +1,190 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebcrawlerClient = void 0;
+const BASE_PATH = "https://api.webcrawlerapi.com";
+const initialPullDelayMs = 2000;
+const MaxPullRetries = 100;
+class WebcrawlerClient {
+    constructor(apiKey, basePath = BASE_PATH, apiVersion = "v1") {
+        this.apiVersion = "v1";
+        this.apiKey = apiKey;
+        this.basePath = basePath;
+        this.apiVersion = apiVersion;
+    }
+    scrapeAsync(scrapeRequest) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.basePath}/${this.apiVersion}/scrape`;
+            const requestOptions = {
+                'method': 'POST',
+                'headers': {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this.apiKey}`
+                },
+                'body': JSON.stringify(scrapeRequest),
+            };
+            const response = yield fetch(url, requestOptions);
+            if (response.ok) {
+                return response.json();
+            }
+            try {
+                const data = yield response.json();
+                throw new Error(`failed to scrape, response returned ${response.status} ${response.statusText}: ${data === null || data === void 0 ? void 0 : data.error}`);
+            }
+            catch (e) {
+                throw e;
+            }
+        });
+    }
+    scrapeWithMeta(scrapeRequest) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.basePath}/${this.apiVersion}/scrape`;
+            const requestOptions = {
+                'method': 'POST',
+                'headers': {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this.apiKey}`
+                },
+                'body': JSON.stringify(scrapeRequest),
+            };
+            const jobIdResponse = yield this.sendRequest(url, requestOptions);
+            if (jobIdResponse.id === '') {
+                throw new Error("Failed to fetch job status");
+            }
+            let delayIntervalMs = initialPullDelayMs;
+            for (let i = 0; i < MaxPullRetries; i++) {
+                yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
+                const scrapeResult = yield this.getScrapeResult(jobIdResponse.id);
+                if (scrapeRequest.debug) {
+                    console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
+                }
+                if (scrapeResult.status !== 'in_progress' && scrapeResult.status !== 'new') {
+                    return scrapeResult;
+                }
+                if (scrapeResult.recommended_pull_delay_ms > 0) {
+                    delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
+                }
+            }
+            throw new Error("Scraping took too long, please retry");
+        });
+    }
+    scrape(scrapeRequest) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const scrapeResult = yield this.scrapeWithMeta(scrapeRequest);
+            return scrapeResult.structured_data;
+        });
+    }
+    getScrapeResult(scrapeID) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.basePath}/${this.apiVersion}/scrape/${scrapeID}`;
+            const requestOptions = {
+                'method': 'GET',
+                'headers': {
+                    'Authorization': `Bearer ${this.apiKey}`,
+                },
+            };
+            const response = yield fetch(url, requestOptions);
+            if (response.ok) {
+                return response.json();
+            }
+            try {
+                const data = yield response.json();
+                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`);
+            }
+            catch (e) {
+                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}`);
+            }
+        });
+    }
+    crawl(crawlRequest) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.basePath}/${this.apiVersion}/crawl`;
+            const requestOptions = {
+                'method': 'POST',
+                'headers': {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this.apiKey}`
+                },
+                'body': JSON.stringify(crawlRequest),
+            };
+            const jobIdResponse = yield this.sendRequest(url, requestOptions);
+            if (jobIdResponse.id === '') {
+                throw new Error("Failed to fetch job status");
+            }
+            let delayIntervalMs = initialPullDelayMs;
+            for (let i = 0; i < MaxPullRetries; i++) {
+                yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
+                const job = yield this.getJob(jobIdResponse.id);
+                if (job.status !== 'in_progress' && job.status !== 'new') {
+                    return job;
+                }
+                if (job.recommended_pull_delay_ms > 0) {
+                    delayIntervalMs = job.recommended_pull_delay_ms;
+                }
+            }
+        });
+    }
+    crawlAsync(crawlRequest) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.basePath}/${this.apiVersion}/crawl`;
+            const requestOptions = {
+                'method': 'POST',
+                'headers': {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this.apiKey}`
+                },
+                'body': JSON.stringify(crawlRequest),
+            };
+            return yield this.sendRequest(url, requestOptions);
+        });
+    }
+    getJob(jobID) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
+            console.log(url);
+            const requestOptions = {
+                'method': 'GET',
+                'headers': {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this.apiKey}`
+                }
+            };
+            const response = yield fetch(url, requestOptions);
+            if (response.ok) {
+                return response.json();
+            }
+            try {
+                const data = yield response.json();
+                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}: ${JSON.stringify(data)}`);
+            }
+            catch (e) {
+                throw new Error(`failed to fetch job status ${response.status} ${response.statusText}`);
+            }
+        });
+    }
+    sendRequest(url, requestOptions) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let response;
+            try {
+                response = yield fetch(url, requestOptions);
+            }
+            catch (e) {
+                throw new Error(`Failed to send request: ${e}`);
+            }
+            if (!response.ok) {
+                const errorResponse = yield response.json();
+                throw new Error(`${JSON.stringify(errorResponse)}`);
+            }
+            return response.json();
+        });
+    }
+}
+exports.WebcrawlerClient = WebcrawlerClient;

package/dist/configuration.d.ts

@@ -0,0 +1,16 @@
+export interface ConfigurationParameters {
+    apiKey?: string;
+    basePath?: string;
+    apiVersion?: string;
+}
+export declare class Configuration {
+    /**
+     * Base options for http call
+     */
+    baseOptions?: any;
+    /**
+     * API version, for example: "v1"
+     */
+    apiVersion: string;
+    constructor(param?: ConfigurationParameters);
+}

package/dist/configuration.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Configuration = void 0;
+const packageJson = require("../package.json");
+class Configuration {
+    constructor(param = {}) {
+        /**
+         * API version, for example: "v1"
+         */
+        this.apiVersion = "v1";
+        this.apiVersion = param.apiVersion;
+        if (!this.baseOptions) {
+            this.baseOptions = {};
+        }
+        this.baseOptions = {
+            headers: Object.assign({ 'User-Agent': `webcrawlerapi-js/${packageJson.version}`, 'Content-Type': 'application/json' }, this.baseOptions.headers)
+        };
+    }
+}
+exports.Configuration = Configuration;

package/dist/index.d.ts

@@ -0,0 +1 @@
+export * from "./api";

package/dist/index.js

@@ -0,0 +1,17 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./api"), exports);

package/dist/model.d.ts

@@ -0,0 +1,58 @@
+export interface ScrapeRequest {
+    input: any;
+    crawler_id: string;
+    max_retries?: number;
+    debug?: boolean;
+}
+export interface JobId {
+    id: string;
+}
+export interface ScrapeResponse {
+    id: string;
+    url: string;
+    status: any;
+    page_status_code: number;
+    created_at: string;
+    structured_data: any;
+    recommended_pull_delay_ms: number;
+}
+export interface CrawlRequest {
+    url: string;
+    items_limit?: number;
+    scrape_type?: string;
+    whitelist_regexp?: string;
+    blacklist_regexp?: string;
+    allow_subdomains?: boolean;
+    webhook_url?: string;
+}
+export interface Job {
+    id: string;
+    org_id: string;
+    url: string;
+    status: string;
+    scrape_type: string;
+    whitelist_regexp: string;
+    blacklist_regexp: string;
+    allow_subdomains: boolean;
+    items_limit: number;
+    created_at: string;
+    finished_at: string;
+    updated_at: string;
+    webhook_url: string;
+    recommended_pull_delay_ms: number;
+    job_items: JobItem[];
+}
+export interface JobItem {
+    id: string;
+    job_id: string;
+    original_url: string;
+    page_status_code: number;
+    markdown_content_url: string;
+    status: string;
+    title: string;
+    last_error: string;
+    created_at: string;
+    updated_at: string;
+    cost: number;
+    referred_url: string;
+}

package/dist/model.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/index.ts

@@ -0,0 +1 @@
+export * from "./api";

package/model.ts

@@ -0,0 +1,63 @@
+export interface ScrapeRequest {
+    input: any;
+    crawler_id: string;
+    max_retries?: number;
+    debug?: boolean;
+}
+
+export interface JobId {
+    id: string;
+}
+
+export interface ScrapeResponse {
+    id: string;
+    url: string;
+    status: any;
+    page_status_code: number;
+    created_at: string;
+    structured_data: any;
+    recommended_pull_delay_ms: number;
+}
+
+export interface CrawlRequest {
+    url: string;
+    items_limit?: number;
+    scrape_type?: string;
+    whitelist_regexp?: string;
+    blacklist_regexp?: string;
+    allow_subdomains?: boolean;
+    webhook_url?: string;
+}
+
+export interface Job {
+    id: string;
+    org_id: string;
+    url: string;
+    status: string;
+    scrape_type: string;
+    whitelist_regexp: string;
+    blacklist_regexp: string;
+    allow_subdomains: boolean;
+    items_limit: number;
+    created_at: string;
+    finished_at: string;
+    updated_at: string;
+    webhook_url: string;
+    recommended_pull_delay_ms: number;
+    job_items: JobItem[];
+}
+
+export interface JobItem {
+    id: string;
+    job_id: string;
+    original_url: string;
+    page_status_code: number;
+    markdown_content_url: string;
+    status: string;
+    title: string;
+    last_error: string;
+    created_at: string;
+    updated_at: string;
+    cost: number;
+    referred_url: string;
+}

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "webcrawlerapi-js",
+  "version": "1.0.4",
+  "description": "JS client for WecrawlerAPI",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "keywords": [
+    "scraping",
+    "scraper",
+    "crawler",
+    "crawling",
+    "webcrawler",
+    "webcrawling",
+    "ai"
+  ],
+  "devDependencies": {
+    "@types/node": "^22.4.0",
+    "typescript": "^5.5.4"
+  },
+  "author": "Andrew <support@webcrawlerapi.com>",
+  "license": "MIT",
+  "dependencies": {},
+  "scripts": {
+    "build": "tsc --outDir dist/"
+  }
+}

package/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "declaration": true,
+    "target": "es6",
+    "module": "commonjs",
+    "noImplicitAny": true,
+    "outDir": "dist",
+    "rootDir": ".",
+    "typeRoots": [
+      "node_modules/@types"
+    ]
+  },
+  "exclude": [
+    "dist",
+    "node_modules"
+  ]
+}