scrapfly-sdk 0.5.1 → 0.6.4

package/esm/screenshotconfig.js

@@ -0,0 +1,220 @@
+import { ScreenshotConfigError } from './errors.js';
+import { urlsafe_b64encode } from './utils.js';
+import { log } from './logger.js';
+export var Options;
+(function (Options) {
+    /**
+      Options to customize the screenshot behavior
+      Attributes:
+          LOAD_IMAGES: Enable image rendering with the request, add extra usage for the bandwidth consumed.
+          DARK_MODE: Enable dark mode display.
+          BLOCK_BANNERS: Block cookies banners and overlay that cover the screen.
+          PRINT_MEDIA_FORMAT: Render the page in the print mode.
+      */
+    Options["LOAD_IMAGES"] = "load_images";
+    Options["DARK_MODE"] = "dark_mode";
+    Options["BLOCK_BANNERS"] = "block_banners";
+    Options["PRINT_MEDIA_FORMAT"] = "print_media_format";
+})(Options || (Options = {}));
+export var Format;
+(function (Format) {
+    /**
+      Format of the screenshot image.
+      Attributes:
+          JPG: JPG format.
+          PNG: PNG format.
+          WEBP: WEBP format.
+          GIF: GIF format.
+      */
+    Format["JPG"] = "jpg";
+    Format["PNG"] = "png";
+    Format["WEBP"] = "webp";
+    Format["GIF"] = "gif";
+})(Format || (Format = {}));
+export class ScreenshotConfig {
+    constructor(options) {
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "format", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "capture", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "resolution", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "country", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "timeout", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "rendering_wait", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "wait_for_selector", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "auto_scroll", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "js", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cache", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cache_ttl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cache_clear", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "webhook", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.validateOptions(options);
+        if (options.format && !Object.values(Format).includes(options.format)) {
+            throw new ScreenshotConfigError(`Invalid format param value: ${options.format}`);
+        }
+        this.format = options.format ?? this.format;
+        // Validate options against the enum
+        if (options.options) {
+            options.options.forEach((opt) => {
+                if (!Object.values(Options).includes(opt)) {
+                    throw new ScreenshotConfigError(`Invalid options param value: ${opt}`);
+                }
+            });
+        }
+        this.url = options.url;
+        this.format = options.format ?? this.format;
+        this.capture = options.capture ?? this.capture;
+        this.resolution = options.resolution ?? this.resolution;
+        this.country = options.country ?? this.country;
+        this.timeout = options.timeout ?? this.timeout;
+        this.rendering_wait = options.rendering_wait ?? this.rendering_wait;
+        this.wait_for_selector = options.wait_for_selector ?? this.wait_for_selector;
+        this.options = options.options ?? this.options;
+        this.auto_scroll = options.auto_scroll ?? this.auto_scroll;
+        this.js = options.js ?? this.js;
+        this.cache = options.cache ?? this.cache;
+        this.cache_ttl = options.cache_ttl ?? this.cache_ttl;
+        this.cache_clear = options.cache_clear ?? this.cache_clear;
+        this.webhook = options.webhook;
+    }
+    validateOptions(options) {
+        const validKeys = new Set(Object.keys(this));
+        for (const key in options) {
+            if (!validKeys.has(key)) {
+                throw new ScreenshotConfigError(`Invalid option provided: ${key}`);
+            }
+        }
+    }
+    toApiParams(options) {
+        const params = {
+            key: options.key,
+        };
+        params.url = this.url;
+        if (this.format) {
+            params.format = this.format.valueOf();
+        }
+        if (this.capture) {
+            params.capture = this.capture;
+        }
+        if (this.resolution) {
+            params.resolution = this.resolution;
+        }
+        if (this.country) {
+            params.country = this.country;
+        }
+        if (this.timeout) {
+            params.timeout = this.timeout;
+        }
+        if (this.rendering_wait) {
+            params.rendering_wait = this.rendering_wait;
+        }
+        if (this.wait_for_selector) {
+            params.wait_for_selector = this.wait_for_selector;
+        }
+        if (this.options) {
+            params.options = this.options.join(',');
+        }
+        if (this.auto_scroll === true) {
+            params.auto_scroll = this.auto_scroll;
+        }
+        if (this.js) {
+            params.js = urlsafe_b64encode(this.js);
+        }
+        if (this.cache === true) {
+            params.cache = this.cache;
+            if (this.cache_ttl) {
+                params.cache_ttl = this.cache_ttl;
+            }
+            if (this.cache_clear === true) {
+                params.cache_clear = this.cache_clear;
+            }
+        }
+        else {
+            if (this.cache_ttl) {
+                log.warn('Params "cache_ttl" is ignored. Works only if cache is enabled');
+            }
+            if (this.cache_clear) {
+                log.warn('Params "cache_clear" is ignored. Works only if cache is enabled');
+            }
+        }
+        if (this.webhook) {
+            params.webhook_name = this.webhook;
+        }
+        return params;
+    }
+}

package/{build/src → esm}/types.d.ts

@@ -1,2 +1,3 @@
 export type Rec<T> = Record<string, T>;
 export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'HEAD';
+//# sourceMappingURL=types.d.ts.map

package/esm/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACvC,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,OAAO,GAAG,MAAM,CAAC"}

package/esm/types.js

@@ -0,0 +1 @@
+export {};

package/esm/utils.d.ts

@@ -0,0 +1,10 @@
+/// <reference types="node" />
+export declare function urlsafe_b64encode(data: string): string;
+export type RequestOptions = {
+    url: string;
+    method?: string;
+    headers?: any;
+    body?: string | Uint8Array;
+};
+export declare function fetchRetry(config: RequestOptions, retries?: number, retryDelay?: number): Promise<Response>;
+//# sourceMappingURL=utils.d.ts.map

package/esm/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAEA,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;CAC5B,CAAC;AAEF,wBAAsB,UAAU,CAC9B,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,MAAU,EACnB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,QAAQ,CAAC,CA+BnB"}

package/esm/utils.js

@@ -0,0 +1,40 @@
+import { log } from './logger.js';
+export function urlsafe_b64encode(data) {
+    const encoder = new TextEncoder();
+    const encoded = encoder.encode(data);
+    const base64 = btoa(String.fromCharCode(...encoded))
+        .replace(/\+/g, '-')
+        .replace(/\//g, '_')
+        .replace(/=+$/, '');
+    return base64;
+}
+export async function fetchRetry(config, retries = 3, retryDelay = 1000) {
+    let lastError = null;
+    for (let attempt = 1; attempt <= retries; attempt++) {
+        try {
+            // XXX: note that cloudflare workers don't support init options
+            const { url, ...reqInit } = config;
+            const response = await fetch(new Request(url, reqInit));
+            // retry 5xx status codes
+            if (response.status >= 500 && response.status < 600) {
+                const _text = await response.text(); // consume response to prevent leak
+                lastError = new Error(`Fetch failed with status: ${response.status}`);
+                if (attempt < retries) {
+                    log.debug(`request failed ${response.status} (${_text}): retry ${attempt}/${retries} after ${retryDelay}ms`);
+                    await new Promise((resolve) => setTimeout(resolve, retryDelay));
+                }
+            }
+            else {
+                return response;
+            }
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt === retries || error.name === 'AbortError') {
+                throw error;
+            }
+            await new Promise((resolve) => setTimeout(resolve, retryDelay));
+        }
+    }
+    throw lastError;
+}

package/package.json

@@ -1,63 +1,38 @@
 {
-    "name": "scrapfly-sdk",
-    "version": "0.5.1",
-    "description": "SDK for Scrapfly.io web scraping service",
-    "type": "module",
-    "types": "build/src/main.d.ts",
-    "main": "build/src/main.js",
-    "repository": {
-        "type": "git",
-        "url": "https://github.com/scrapfly/typescript-scrapfly"
-    },
-    "bugs": "https://github.com/scrapfly/typescript-scrapfly/issues",
-    "homepage": "https://scrapfly.io/",
-    "keywords": [
-        "web scraping",
-        "SDK",
-        "scrapfly",
-        "api"
-    ],
-    "files": [
-        "build/src"
-    ],
-    "engines": {
-        "node": ">= 18 <19"
-    },
-    "devDependencies": {
-        "@types/node": "~18",
-        "@typescript-eslint/eslint-plugin": "~5.59",
-        "@typescript-eslint/parser": "~5.59",
-        "eslint": "~8.38",
-        "eslint-config-prettier": "~8.8",
-        "eslint-plugin-jest": "~27.2",
-        "jest": "~29.5",
-        "prettier": "~2.8",
-        "rimraf": "~5.0",
-        "ts-api-utils": "~0.0.44",
-        "ts-jest": "~29.1",
-        "typescript": "^5.1.6"
-    },
-    "scripts": {
-        "clean": "rimraf coverage build tmp",
-        "prebuild": "npm run lint",
-        "build": "tsc -p tsconfig.json",
-        "build:watch": "tsc -w -p tsconfig.json",
-        "build:release": "npm run clean && tsc -p tsconfig.release.json",
-        "lint": "eslint . --ext .ts --ext .mts",
-        "test": "jest --coverage",
-        "prettier": "prettier --config .prettierrc --write .",
-        "test:watch": "jest --watch"
-    },
-    "author": "Bernardas Alisauskas <bernardas@scrapfly.io>",
-    "license": "BSD",
-    "dependencies": {
-        "@types/cheerio": "^0.22.31",
-        "axios": "^1.4.0",
-        "axios-retry": "^4.0.0",
-        "cheerio": "^1.0.0-rc.12",
-        "tslib": "~2.5"
-    },
-    "volta": {
-        "node": "18.17.0"
+  "name": "scrapfly-sdk",
+  "version": "0.6.4",
+  "description": "SDK for Scrapfly.io API for web scraping, screenshotting and data extraction",
+  "keywords": [
+    "web scraping",
+    "scrapfly",
+    "api",
+    "sdk"
+  ],
+  "homepage": "https://scrapfly.io/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/scrapfly/typescript-scrapfly.git"
+  },
+  "license": "BSD",
+  "bugs": {
+    "url": "https://github.com/scrapfly/typescript-scrapfly/issues"
+  },
+  "main": "./esm/main.js",
+  "module": "./esm/main.js",
+  "types": "./esm/main.d.ts",
+  "exports": {
+    ".": {
+      "import": "./esm/main.js",
+      "require": "./script/main.js"
     }
-}
+  },
+  "type": "module",
+  "dependencies": {
+    "cheerio": "1.0.0-rc.12",
+    "@deno/shim-deno": "~0.18.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.9.0"
+  },
+  "_generatedBy": "dnt@dev"
+}

package/script/_dnt.shims.d.ts

@@ -0,0 +1,6 @@
+import { Deno } from "@deno/shim-deno";
+export { Deno } from "@deno/shim-deno";
+export declare const dntGlobalThis: Omit<typeof globalThis, "Deno"> & {
+    Deno: typeof Deno;
+};
+//# sourceMappingURL=_dnt.shims.d.ts.map

package/script/_dnt.shims.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"_dnt.shims.d.ts","sourceRoot":"","sources":["../src/_dnt.shims.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAKvC,eAAO,MAAM,aAAa;;CAA2C,CAAC"}

package/script/_dnt.shims.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dntGlobalThis = exports.Deno = void 0;
+const shim_deno_1 = require("@deno/shim-deno");
+var shim_deno_2 = require("@deno/shim-deno");
+Object.defineProperty(exports, "Deno", { enumerable: true, get: function () { return shim_deno_2.Deno; } });
+const dntGlobals = {
+    Deno: shim_deno_1.Deno,
+};
+exports.dntGlobalThis = createMergeProxy(globalThis, dntGlobals);
+function createMergeProxy(baseObj, extObj) {
+    return new Proxy(baseObj, {
+        get(_target, prop, _receiver) {
+            if (prop in extObj) {
+                return extObj[prop];
+            }
+            else {
+                return baseObj[prop];
+            }
+        },
+        set(_target, prop, value) {
+            if (prop in extObj) {
+                delete extObj[prop];
+            }
+            baseObj[prop] = value;
+            return true;
+        },
+        deleteProperty(_target, prop) {
+            let success = false;
+            if (prop in extObj) {
+                delete extObj[prop];
+                success = true;
+            }
+            if (prop in baseObj) {
+                delete baseObj[prop];
+                success = true;
+            }
+            return success;
+        },
+        ownKeys(_target) {
+            const baseKeys = Reflect.ownKeys(baseObj);
+            const extKeys = Reflect.ownKeys(extObj);
+            const extKeysSet = new Set(extKeys);
+            return [...baseKeys.filter((k) => !extKeysSet.has(k)), ...extKeys];
+        },
+        defineProperty(_target, prop, desc) {
+            if (prop in extObj) {
+                delete extObj[prop];
+            }
+            Reflect.defineProperty(baseObj, prop, desc);
+            return true;
+        },
+        getOwnPropertyDescriptor(_target, prop) {
+            if (prop in extObj) {
+                return Reflect.getOwnPropertyDescriptor(extObj, prop);
+            }
+            else {
+                return Reflect.getOwnPropertyDescriptor(baseObj, prop);
+            }
+        },
+        has(_target, prop) {
+            return prop in extObj || prop in baseObj;
+        },
+    });
+}

package/script/client.d.ts

@@ -0,0 +1,70 @@
+/// <reference types="node" />
+import { fetchRetry } from './utils.js';
+import * as errors from './errors.js';
+import type { ScrapeConfig } from './scrapeconfig.js';
+import type { ScreenshotConfig } from './screenshotconfig.js';
+import type { ExtractionConfig } from './extractionconfig.js';
+import { type AccountData, ExtractionResult, ScrapeResult, ScreenshotResult } from './result.js';
+export declare class ScrapflyClient {
+    HOST: string;
+    private key;
+    private ua;
+    fetch: typeof fetchRetry;
+    constructor(options: {
+        key: string;
+    });
+    /**
+     * Raise appropriate error for given response and scrape result
+     */
+    errResult(response: Response, result: ScrapeResult): errors.ScrapflyError;
+    /**
+     * Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
+     */
+    handleResponse(response: Response, result: ScrapeResult): ScrapeResult;
+    /**
+     * Retrieve Scrapfly account details
+     */
+    account(): Promise<AccountData>;
+    /**
+     * Issue a single scrape command from a given scrape configuration
+     */
+    scrape(config: ScrapeConfig): Promise<ScrapeResult>;
+    /**
+    Concurrently scrape multiple configs
+    This is a async generator call it like this:
+  
+      const results = [];
+      const errors = [];
+      for await (const resultOrError of client.concurrentScrape(configs)) {
+          if (resultOrError instanceof Error) {
+              errors.push(resultOrError);
+          } else {
+              results.push(resultOrError);
+          }
+      }
+  
+    @param concurrencyLimit: if not set it will be taken from your account info
+    */
+    concurrentScrape(configs: ScrapeConfig[], concurrencyLimit?: number): AsyncGenerator<ScrapeResult | Error | undefined, void, undefined>;
+    /**
+     * Save screenshot response to a file
+     */
+    saveScreenshot(result: ScreenshotResult, name: string, savePath?: string): Promise<any>;
+    /**
+     * Turn scrapfly screenshot API response to ScreenshotResult or raise one of ScrapflyError
+     */
+    handleScreenshotResponse(response: Response): Promise<ScreenshotResult>;
+    /**
+     * Take a screenshot
+     */
+    screenshot(config: ScreenshotConfig): Promise<ScreenshotResult>;
+    /**
+     * Turn scrapfly Extraction API response to ExtractionResult or raise one of ScrapflyError
+     */
+    handleExtractionResponse(response: Response): Promise<ExtractionResult>;
+    /**
+     * Extract structured data from a web page
+     */
+    extract(config: ExtractionConfig): Promise<ExtractionResult>;
+}
+//# sourceMappingURL=client.d.ts.map

package/script/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA4CzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}