scrapfly-sdk 0.5.1 → 0.6.4

package/script/scrapeconfig.js

@@ -0,0 +1,509 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScrapeConfig = exports.FormatOption = exports.Format = exports.ScreenshotFlags = void 0;
+const utils_js_1 = require("./utils.js");
+const logger_js_1 = require("./logger.js");
+const errors_js_1 = require("./errors.js");
+var ScreenshotFlags;
+(function (ScreenshotFlags) {
+    /**
+      Options to customize the screenshot behavior
+      Attributes:
+          LOAD_IMAGES: Enable image rendering with the request, add extra usage for the bandwidth consumed.
+          DARK_MODE: Enable dark mode display.
+          BLOCK_BANNERS: Block cookies banners and overlay that cover the screen.
+          PRINT_MEDIA_FORMAT: Render the page in the print mode.
+      */
+    ScreenshotFlags["LOAD_IMAGES"] = "load_images";
+    ScreenshotFlags["DARK_MODE"] = "dark_mode";
+    ScreenshotFlags["BLOCK_BANNERS"] = "block_banners";
+    ScreenshotFlags["PRINT_MEDIA_FORMAT"] = "print_media_format";
+    ScreenshotFlags["HIGH_QUALITY"] = "high_quality";
+})(ScreenshotFlags || (exports.ScreenshotFlags = ScreenshotFlags = {}));
+var Format;
+(function (Format) {
+    /**
+      Format of the scraped content.
+      Attributes:
+          JSON: JSON format.
+          TEXT: Text format.
+          MARKDOWN: Markdown format.
+          CLEAN_HTML: Clean HTML format.
+      */
+    Format["JSON"] = "json";
+    Format["TEXT"] = "text";
+    Format["MARKDOWN"] = "markdown";
+    Format["CLEAN_HTML"] = "clean_html";
+    Format["RAW"] = "raw";
+})(Format || (exports.Format = Format = {}));
+var FormatOption;
+(function (FormatOption) {
+    FormatOption["NO_LINKS"] = "no_links";
+    FormatOption["NO_IMAGES"] = "no_images";
+})(FormatOption || (exports.FormatOption = FormatOption = {}));
+class ScrapeConfig {
+    constructor(options) {
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "retry", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        Object.defineProperty(this, "method", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 'GET'
+        });
+        Object.defineProperty(this, "country", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "render_js", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "cache", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "cache_clear", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "cost_budget", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "ssl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "dns", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "asp", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "debug", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "raise_on_upstream_error", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        Object.defineProperty(this, "cache_ttl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "proxy_pool", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "session", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "tags", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Set()
+        });
+        Object.defineProperty(this, "format", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        }); // raw(unchanged)
+        Object.defineProperty(this, "format_options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "correlation_id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cookies", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "body", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "data", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "headers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "js", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "rendering_wait", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "wait_for_selector", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "session_sticky_proxy", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "screenshots", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "screenshot_flags", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "webhook", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "timeout", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        }); // in milliseconds
+        Object.defineProperty(this, "js_scenario", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "lang", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "os", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "auto_scroll", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.validateOptions(options);
+        if (options.format && !Object.values(Format).includes(options.format)) {
+            throw new errors_js_1.ScrapeConfigError(`Invalid format param value: ${options.format}`);
+        }
+        this.format = options.format ?? this.format;
+        if (options.screenshot_flags) {
+            options.screenshot_flags.forEach((flag) => {
+                if (!Object.values(ScreenshotFlags).includes(flag)) {
+                    throw new errors_js_1.ScrapeConfigError(`Invalid screenshot_flags param value: ${flag}`);
+                }
+            });
+        }
+        this.url = options.url;
+        this.retry = options.retry ?? this.retry;
+        this.method = options.method ?? this.method;
+        this.country = options.country ?? this.country;
+        this.session_sticky_proxy = options.session_sticky_proxy ?? this.session_sticky_proxy;
+        this.render_js = options.render_js ?? this.render_js;
+        this.cache = options.cache ?? this.cache;
+        this.cache_clear = options.cache_clear ?? this.cache_clear;
+        this.cost_budget = options.cost_budget ?? this.cost_budget;
+        this.asp = options.asp ?? this.asp;
+        this.headers = options.headers
+            ? Object.fromEntries(Object.entries(options.headers).map(([k, v]) => [k.toLowerCase(), v]))
+            : {};
+        this.raise_on_upstream_error = options.raise_on_upstream_error ?? this.raise_on_upstream_error;
+        this.cache_ttl = options.cache_ttl ?? this.cache_ttl;
+        this.proxy_pool = options.proxy_pool ?? this.proxy_pool;
+        this.session = options.session ?? this.session;
+        this.tags = new Set(options.tags) ?? this.tags;
+        this.format = options.format ?? this.format;
+        this.format_options = options.format_options ?? this.format_options;
+        this.correlation_id = options.correlation_id ?? this.correlation_id;
+        this.cookies = options.cookies
+            ? Object.fromEntries(Object.entries(options.cookies).map(([k, v]) => [k.toLowerCase(), v]))
+            : {};
+        this.body = options.body ?? this.body;
+        this.data = options.data ?? this.data;
+        this.js = options.js ?? this.js;
+        this.rendering_wait = options.rendering_wait ?? this.rendering_wait;
+        this.wait_for_selector = options.wait_for_selector ?? this.wait_for_selector;
+        this.screenshots = options.screenshots ?? this.screenshots;
+        this.screenshot_flags = options.screenshot_flags ?? this.screenshot_flags;
+        this.webhook = options.webhook ?? this.webhook;
+        this.timeout = options.timeout ?? this.timeout;
+        this.js_scenario = options.js_scenario ?? this.js_scenario;
+        this.os = options.os ?? this.os;
+        this.lang = options.lang ?? this.lang;
+        this.auto_scroll = options.auto_scroll ?? this.auto_scroll;
+        this.dns = options.dns ?? this.dns;
+        this.ssl = options.ssl ?? this.ssl;
+        this.debug = options.debug ?? this.debug;
+        if (this.body && this.data) {
+            throw new errors_js_1.ScrapeConfigError('Cannot set both body and data');
+        }
+        if (['POST', 'PUT', 'PATCH'].includes(this.method)) {
+            if (this.data && !this.body) {
+                if (!this.headers['content-type']) {
+                    this.headers['content-type'] = 'application/x-www-form-urlencoded';
+                    this.body = new URLSearchParams(this.data).toString();
+                }
+                else {
+                    if (this.headers['content-type']?.includes('application/json')) {
+                        this.body = JSON.stringify(this.data);
+                    }
+                    else if (this.headers['content-type']?.includes('application/x-www-form-urlencoded')) {
+                        this.body = new URLSearchParams(this.data).toString();
+                    }
+                    else {
+                        throw new errors_js_1.ScrapeConfigError(`Content-Type "${this.headers['content-type']}" not supported, use body parameter to pass pre encoded body according to your content type`);
+                    }
+                }
+            }
+            else if (this.body && !this.data) {
+                this.headers['content-type'] = 'text/plain';
+            }
+        }
+    }
+    validateOptions(options) {
+        const validKeys = new Set(Object.keys(this));
+        for (const key in options) {
+            if (!validKeys.has(key)) {
+                throw new errors_js_1.ScrapeConfigError(`Invalid option provided: ${key}`);
+            }
+        }
+    }
+    toApiParams(options) {
+        const params = {
+            key: options.key,
+        };
+        params.url = this.url;
+        if (this.country) {
+            params.country = this.country;
+        }
+        if (this.headers !== undefined && Object.keys(this.headers).length > 0) {
+            Object.entries(this.headers).forEach(([key, value]) => {
+                params[`headers[${key}]`] = value;
+            });
+        }
+        if (this.cookies !== undefined && Object.keys(this.cookies).length > 0) {
+            const cookiesAsHeader = [...Object.entries(this.cookies)]
+                .map(([key, value]) => `${key}=${value}`)
+                .join('; ');
+            if (params['headers[cookie]']) {
+                // if current cookie value doesn't have a ';' at the end, add it.
+                if (params['headers[cookie]'][params['headers[cookie]'].length - 1] !== ';') {
+                    params['headers[cookie]'] += ';';
+                }
+                params['headers[cookie]'] += ` ${cookiesAsHeader}`;
+            }
+            else {
+                params['headers[cookie]'] = cookiesAsHeader;
+            }
+        }
+        if (this.webhook) {
+            params.webhook_name = this.webhook;
+        }
+        if (this.timeout) {
+            params.timeout = this.timeout;
+        }
+        if (this.render_js === true) {
+            params.render_js = true;
+            if (this.wait_for_selector !== undefined) {
+                params.wait_for_selector = this.wait_for_selector;
+            }
+            if (this.js !== undefined) {
+                params.js = (0, utils_js_1.urlsafe_b64encode)(this.js);
+            }
+            if (this.js_scenario !== undefined) {
+                params.js_scenario = (0, utils_js_1.urlsafe_b64encode)(JSON.stringify(this.js_scenario));
+            }
+            if (this.rendering_wait !== undefined) {
+                params.rendering_wait = this.rendering_wait;
+            }
+            if (this.screenshots !== undefined) {
+                Object.keys(this.screenshots).forEach((key) => {
+                    params[`screenshots[${key}]`] = (this.screenshots || {})[key];
+                });
+                if (this.screenshot_flags) {
+                    params.screenshot_flags = this.screenshot_flags.join(',');
+                }
+            }
+            else {
+                if (this.screenshot_flags) {
+                    logger_js_1.log.warn('Params "screenshot_flags" is ignored. Works only if screenshots is enabled');
+                }
+            }
+            if (this.auto_scroll !== undefined) {
+                params.auto_scroll = this.auto_scroll;
+            }
+        }
+        else {
+            if (this.wait_for_selector !== undefined) {
+                logger_js_1.log.warn('Params "wait_for_selector" is ignored. Works only if render_js is enabled');
+            }
+            if (this.screenshots !== undefined) {
+                logger_js_1.log.warn('Params "screenshots" is ignored. Works only if render_js is enabled');
+            }
+            if (this.js_scenario !== undefined) {
+                logger_js_1.log.warn('Params "js_scenario" is ignored. Works only if render_js is enabled');
+            }
+            if (this.js !== undefined) {
+                logger_js_1.log.warn('Params "js" is ignored. Works only if render_js is enabled');
+            }
+            if (this.rendering_wait !== undefined) {
+                logger_js_1.log.warn('Params "rendering_wait" is ignored. Works only if render_js is enabled');
+            }
+        }
+        if (this.asp === true) {
+            params.asp = true;
+        }
+        if (this.retry === false) {
+            params.retry = false;
+        }
+        if (this.cache === true) {
+            params.cache = true;
+            if (this.cache_clear) {
+                params.cache_clear = true;
+            }
+            if (this.cache_ttl) {
+                params.cache_ttl = this.cache_ttl;
+            }
+        }
+        else {
+            if (this.cache_clear) {
+                logger_js_1.log.warn('Params "cache_clear" is ignored. Works only if cache is enabled');
+            }
+            if (this.cache_ttl) {
+                logger_js_1.log.warn('Params "cache_ttl" is ignored. Works only if cache is enabled');
+            }
+        }
+        if (this.dns === true) {
+            params.dns = true;
+        }
+        if (this.ssl === true) {
+            params.ssl = true;
+        }
+        if (this.tags.size > 0) {
+            params.tags = Array.from(this.tags).join(',');
+        }
+        if (this.format) {
+            params.format = this.format.valueOf();
+            if (this.format_options) {
+                params.format += ':' + this.format_options.join(',');
+            }
+        }
+        if (this.correlation_id) {
+            params.correlation_id = this.correlation_id;
+        }
+        if (this.session) {
+            params.session = this.session;
+            if (this.session_sticky_proxy) {
+                params.session_sticky_proxy = true;
+            }
+        }
+        else {
+            if (this.session_sticky_proxy) {
+                logger_js_1.log.warn('Params "session_sticky_proxy" is ignored. Works only if session is enabled');
+            }
+        }
+        if (this.debug === true) {
+            params.debug = true;
+        }
+        if (this.proxy_pool) {
+            params.proxy_pool = this.proxy_pool;
+        }
+        if (this.lang !== undefined) {
+            params.lang = this.lang.join(',');
+        }
+        if (this.os !== undefined) {
+            params.os = this.os;
+        }
+        return params;
+    }
+}
+exports.ScrapeConfig = ScrapeConfig;
+Object.defineProperty(ScrapeConfig, "PUBLIC_DATACENTER_POOL", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: 'public_datacenter_pool'
+});
+Object.defineProperty(ScrapeConfig, "PUBLIC_RESIDENTIAL_POOL", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: 'public_residential_pool'
+});

package/script/screenshotconfig.d.ts

@@ -0,0 +1,69 @@
+export declare enum Options {
+    /**
+      Options to customize the screenshot behavior
+      Attributes:
+          LOAD_IMAGES: Enable image rendering with the request, add extra usage for the bandwidth consumed.
+          DARK_MODE: Enable dark mode display.
+          BLOCK_BANNERS: Block cookies banners and overlay that cover the screen.
+          PRINT_MEDIA_FORMAT: Render the page in the print mode.
+      */
+    LOAD_IMAGES = "load_images",
+    DARK_MODE = "dark_mode",
+    BLOCK_BANNERS = "block_banners",
+    PRINT_MEDIA_FORMAT = "print_media_format"
+}
+export declare enum Format {
+    /**
+      Format of the screenshot image.
+      Attributes:
+          JPG: JPG format.
+          PNG: PNG format.
+          WEBP: WEBP format.
+          GIF: GIF format.
+      */
+    JPG = "jpg",
+    PNG = "png",
+    WEBP = "webp",
+    GIF = "gif"
+}
+type ScreenshotConfigOptions = {
+    url: string;
+    format?: Format;
+    capture?: string;
+    resolution?: string;
+    country?: string;
+    timeout?: number;
+    rendering_wait?: number;
+    wait_for_selector?: string;
+    options?: Options[];
+    auto_scroll?: boolean;
+    js?: string;
+    cache?: boolean;
+    cache_ttl?: number;
+    cache_clear?: boolean;
+    webhook?: string;
+};
+export declare class ScreenshotConfig {
+    url: string;
+    format?: Format;
+    capture?: string;
+    resolution?: string;
+    country?: string;
+    timeout?: number;
+    rendering_wait?: number;
+    wait_for_selector?: string;
+    options?: Options[];
+    auto_scroll?: boolean;
+    js?: string;
+    cache?: boolean;
+    cache_ttl?: number;
+    cache_clear?: boolean;
+    webhook?: string;
+    constructor(options: ScreenshotConfigOptions);
+    private validateOptions;
+    toApiParams(options: {
+        key: string;
+    }): Record<string, any>;
+}
+export {};
+//# sourceMappingURL=screenshotconfig.d.ts.map

package/script/screenshotconfig.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"screenshotconfig.d.ts","sourceRoot":"","sources":["../src/screenshotconfig.ts"],"names":[],"mappings":"AAIA,oBAAY,OAAO;IACjB;;;;;;;QAOI;IACJ,WAAW,gBAAgB;IAC3B,SAAS,cAAc;IACvB,aAAa,kBAAkB;IAC/B,kBAAkB,uBAAuB;CAC1C;AAED,oBAAY,MAAM;IAChB;;;;;;;QAOI;IACJ,GAAG,QAAQ;IACX,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,GAAG,QAAQ;CACZ;AAED,KAAK,uBAAuB,GAAG;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,qBAAa,gBAAgB;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAa;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;gBAEL,OAAO,EAAE,uBAAuB;IA+B5C,OAAO,CAAC,eAAe;IASvB,WAAW,CAAC,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAqE3D"}