zenrows 1.0.4 → 1.1.0

package/README.md

@@ -1,5 +1,5 @@
 # ZenRows Node.js SDK
-SDK to access [ZenRows](https://www.zenrows.com/) API directly from Node.js. We handle proxies rotation, headless browsers, and CAPTCHAs for you.
+SDK to access [ZenRows](https://www.zenrows.com/) API directly from Node.js. ZenRows handles proxies rotation, headless browsers, and CAPTCHAs for you.
 
 ## Installation
 Install the SDK with npm.
@@ -13,8 +13,10 @@ Start using the API by [creating your API Key](https://app.zenrows.com/register?
 
 The SDK uses [axios](https://axios-http.com/) for HTTP requests. The client's response will be an `AxiosPromise` if using TypeScript, a regular `Promise` otherwise.
 
+It also uses [axios-retry](https://github.com/softonic/axios-retry) to automatically retry failed requests (status code 429 and 5XX). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests.
+
 ```javascript
-const { ZenRows } = require('zenrows');
+const { ZenRows } = require('zenrows', { retries: 1 });
 
 const apiKey = 'YOUR-API-KEY';
 const url = 'https://www.zenrows.com/';
@@ -37,6 +39,30 @@ const url = 'https://www.zenrows.com/';
 
         // Make your request from a given country. Requires premium_proxy
         proxy_country: '',
+
+        // Wait for a given CSS Selector to load in the DOM. Requires js_render
+        wait_for: '.content',
+
+        // Wait a fixed amount of time in milliseconds. Requires js_render
+        wait: 2500,
+
+        // Block specific resources from loading, check docs for the full list. Requires js_render
+        block_resources: 'image,media,font',
+
+        // Change the browser's window width and height. Requires js_render
+        window_width: 1920,
+        window_height: 1080,
+
+        // Will automatically use either desktop or mobile user agents in the headers
+        device: 'desktop',
+
+        // Will return the status code returned by the website
+        original_status: false,
+    }, {
+        headers: {
+            Referrer: 'https://www.google.com',
+            'User-Agent': 'MyCustomUserAgent',
+        },
     });
 
     console.log(data);
@@ -56,6 +82,10 @@ const url = 'https://www.zenrows.com/';
 })();
 ```
 
+You can also pass optional parameters and headers; the list above is a reference. For more info, check out [the documentation page](https://www.zenrows.com/documentation).
+
+Sending headers to the target URL will overwrite our defaults. Be careful when doing it and contact us if there is any problem.
+
 ## Examples
 
 Take a look at the [examples directory](./examples) for Javascript and TypeScript files using the SDK.

package/dist/index.d.ts

@@ -1,15 +1,33 @@
 import { AxiosPromise } from 'axios';
+interface ClientConfig {
+    retries?: number;
+}
 interface Config {
     autoparse?: boolean;
     css_extractor?: string;
     js_render?: boolean;
     premium_proxy?: boolean;
     proxy_country?: string;
+    wait_for?: string;
+    wait?: number;
+    block_resources?: string;
+    window_width?: number;
+    window_height?: number;
+    device?: string;
+    original_status?: boolean;
+    custom_headers?: boolean;
     [x: string]: unknown;
 }
+interface Headers {
+    [x: string]: string;
+}
 declare class ZenRows {
     readonly apiKey: string;
-    constructor(apiKey: string);
-    get(url: string, config?: Config): AxiosPromise;
+    readonly clientConfig: ClientConfig;
+    constructor(apiKey: string, clientConfig?: ClientConfig);
+    get(url: string, config?: Config, { headers }?: {
+        headers?: Headers;
+    }): AxiosPromise;
+    private applyRetries;
 }
 export { ZenRows };

package/dist/index.js

@@ -2,23 +2,44 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ZenRows = void 0;
 const axios_1 = require("axios");
+const axios_retry_1 = require("axios-retry");
 const version_1 = require("./version");
 const API_URL = 'https://api.zenrows.com/v1/';
 class ZenRows {
-    constructor(apiKey) {
+    constructor(apiKey, clientConfig = {}) {
         this.apiKey = apiKey;
+        this.clientConfig = clientConfig;
     }
-    get(url, config) {
+    get(url, config, { headers = {} } = {}) {
         const params = Object.assign(Object.assign({}, config), { url, apikey: this.apiKey });
-        const headers = {
-            'User-Agent': `zenrows/${version_1.VERSION} node`,
-        };
+        const finalHeaders = Object.assign({ 'User-Agent': `zenrows/${version_1.VERSION} node` }, headers);
         const axiosRequestConfig = {
             baseURL: API_URL,
             params,
-            headers,
+            headers: finalHeaders,
         };
+        if (headers && Object.keys(headers).length) {
+            params.custom_headers = true;
+        }
+        this.applyRetries();
         return (0, axios_1.default)(axiosRequestConfig);
     }
+    applyRetries() {
+        var _a;
+        const retries = (_a = this.clientConfig.retries) !== null && _a !== void 0 ? _a : 0;
+        if (retries > 0) {
+            (0, axios_retry_1.default)(axios_1.default, {
+                retries,
+                retryDelay: axios_retry_1.default.exponentialDelay,
+                retryCondition: (error) => {
+                    var _a;
+                    if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
+                        return true;
+                    }
+                    return axios_retry_1.default.isNetworkOrIdempotentRequestError(error);
+                },
+            });
+        }
+    }
 }
 exports.ZenRows = ZenRows;

package/dist/queue.d.ts

@@ -0,0 +1,4 @@
+declare const queue: (concurrency: number) => {
+    enqueue: Function;
+};
+export { queue };

package/dist/queue.js

@@ -0,0 +1,37 @@
+"use strict";
+// TODO review disabled!!!!
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.queue = void 0;
+/* eslint-disable @typescript-eslint/ban-types */
+const queue = (concurrency) => {
+    let running = 0;
+    const tasks = [];
+    return {
+        enqueue: (givenTask, ...givenParams) => __awaiter(void 0, void 0, void 0, function* () {
+            tasks.push({ task: givenTask, params: givenParams });
+            if (running >= concurrency) {
+                return;
+            }
+            running += 1;
+            while (tasks.length > 0) {
+                const nextTask = tasks.shift();
+                if (nextTask) {
+                    const { task, params } = nextTask;
+                    // eslint-disable-next-line no-await-in-loop
+                    yield task(...params);
+                }
+            }
+            running -= 1;
+        }),
+    };
+};
+exports.queue = queue;

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "1.0.4";
+export declare const VERSION = "1.1.0";

package/dist/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '1.0.4';
+exports.VERSION = '1.1.0';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zenrows",
-  "version": "1.0.4",
+  "version": "1.1.0",
   "description": "ZenRows Node SDK",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -40,6 +40,7 @@
     "typescript": "^4.4.2"
   },
   "dependencies": {
-    "axios": "^0.21.4"
+    "axios": "^0.21.4",
+    "axios-retry": "^3.2.4"
   }
 }