zenrows 1.0.5 → 1.2.0

package/README.md

@@ -13,8 +13,10 @@ Start using the API by [creating your API Key](https://app.zenrows.com/register?
 
 The SDK uses [axios](https://axios-http.com/) for HTTP requests. The client's response will be an `AxiosPromise` if using TypeScript, a regular `Promise` otherwise.
 
+It also uses [axios-retry](https://github.com/softonic/axios-retry) to automatically retry failed requests (status code 429 and 5XX). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests.
+
 ```javascript
-const { ZenRows } = require('zenrows');
+const { ZenRows } = require('zenrows', { retries: 1 });
 
 const apiKey = 'YOUR-API-KEY';
 const url = 'https://www.zenrows.com/';
@@ -37,6 +39,30 @@ const url = 'https://www.zenrows.com/';
 
         // Make your request from a given country. Requires premium_proxy
         proxy_country: '',
+
+        // Wait for a given CSS Selector to load in the DOM. Requires js_render
+        wait_for: '.content',
+
+        // Wait a fixed amount of time in milliseconds. Requires js_render
+        wait: 2500,
+
+        // Block specific resources from loading, check docs for the full list. Requires js_render
+        block_resources: 'image,media,font',
+
+        // Change the browser's window width and height. Requires js_render
+        window_width: 1920,
+        window_height: 1080,
+
+        // Will automatically use either desktop or mobile user agents in the headers
+        device: 'desktop',
+
+        // Will return the status code returned by the website
+        original_status: false,
+    }, {
+        headers: {
+            Referrer: 'https://www.google.com',
+            'User-Agent': 'MyCustomUserAgent',
+        },
     });
 
     console.log(data);
@@ -56,6 +82,50 @@ const url = 'https://www.zenrows.com/';
 })();
 ```
 
+You can also pass optional parameters and headers; the list above is a reference. For more info, check out [the documentation page](https://www.zenrows.com/documentation).
+
+Sending headers to the target URL will overwrite our defaults. Be careful when doing it and contact us if there is any problem.
+
+### Concurrency
+
+To limit the concurrency, it uses [fastq](https://github.com/mcollina/fastq), which will simultaneously send a maximum of requests. The concurrency is determined by the plan you are in, so take a look at the [pricing](https://www.zenrows.com/pricing) and set it accordingly. Take into account that each client instance will have its own limit, meaning that two different scripts will not share it, and 429 (Too Many Requests) errors might arise.
+
+We use [`Promise.allSettled()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/allSettled) in the example below, available from Node 12.9. It will wait for all the promises to finish, and the results are objects with a status marking them as fulfilled or rejected. The main difference with `Promise.all()` is that it won't fail if any requests fail. It might make your scraping more robust since the whole list of URLs will run, even if some of them fail.
+
+```javascript
+const { ZenRows } = require('zenrows');
+
+const apiKey = 'YOUR-API-KEY';
+
+(async () => {
+    const client = new ZenRows(apiKey, { concurrency: 5, retries: 1 });
+
+    const urls = [
+        'https://www.zenrows.com/',
+        // ...
+    ];
+
+    const promises = urls.map(url => client.get(url));
+
+    const results = await Promise.allSettled(promises);
+    console.log(results);
+    /*
+    [
+        {
+            status: 'fulfilled',
+            value: {
+                status: 200,
+                statusText: 'OK',
+                data: `<!doctype html> <html lang="en"> <head> ...
+            
+        ...
+    */
+
+    // separate results list into rejected and fulfilled for later processing
+    const rejected = results.filter(({ status }) => status === 'rejected');
+    const fulfilled = results.filter(({ status }) => status === 'fulfilled');
+})();
+```
 ## Examples
 
 Take a look at the [examples directory](./examples) for Javascript and TypeScript files using the SDK.

package/dist/index.d.ts

@@ -1,15 +1,41 @@
 import { AxiosPromise } from 'axios';
+import * as fastq from 'fastq';
+interface ClientConfig {
+    concurrency?: number;
+    retries?: number;
+}
 interface Config {
     autoparse?: boolean;
     css_extractor?: string;
     js_render?: boolean;
     premium_proxy?: boolean;
     proxy_country?: string;
+    wait_for?: string;
+    wait?: number;
+    block_resources?: string;
+    window_width?: number;
+    window_height?: number;
+    device?: string;
+    original_status?: boolean;
+    custom_headers?: boolean;
     [x: string]: unknown;
 }
+interface Headers {
+    [x: string]: string;
+}
 declare class ZenRows {
     readonly apiKey: string;
-    constructor(apiKey: string);
-    get(url: string, config?: Config): AxiosPromise;
+    readonly clientConfig: ClientConfig;
+    readonly queue: fastq.queueAsPromised<{
+        url: string;
+        config?: Config | undefined;
+        headers: Headers;
+    }, import("axios").AxiosResponse<any>>;
+    constructor(apiKey: string, clientConfig?: ClientConfig);
+    get(url: string, config?: Config, { headers }?: {
+        headers?: Headers;
+    }): AxiosPromise;
+    private worker;
+    private applyRetries;
 }
 export { ZenRows };

package/dist/index.js

@@ -2,23 +2,50 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ZenRows = void 0;
 const axios_1 = require("axios");
+const axios_retry_1 = require("axios-retry");
+const fastq = require("fastq");
 const version_1 = require("./version");
 const API_URL = 'https://api.zenrows.com/v1/';
 class ZenRows {
-    constructor(apiKey) {
+    constructor(apiKey, clientConfig = {}) {
+        var _a;
         this.apiKey = apiKey;
+        this.clientConfig = clientConfig;
+        this.queue = fastq.promise(this, this.worker, (_a = this.clientConfig.concurrency) !== null && _a !== void 0 ? _a : 5);
+        this.applyRetries();
     }
-    get(url, config) {
+    get(url, config, { headers = {} } = {}) {
+        return this.queue.push({ url, config, headers });
+    }
+    worker({ url, config, headers }) {
         const params = Object.assign(Object.assign({}, config), { url, apikey: this.apiKey });
-        const headers = {
-            'User-Agent': `zenrows/${version_1.VERSION} node`,
-        };
+        const finalHeaders = Object.assign({ 'User-Agent': `zenrows/${version_1.VERSION} node` }, headers);
         const axiosRequestConfig = {
             baseURL: API_URL,
             params,
-            headers,
+            headers: finalHeaders,
         };
+        if (headers && Object.keys(headers).length) {
+            params.custom_headers = true;
+        }
         return (0, axios_1.default)(axiosRequestConfig);
     }
+    applyRetries() {
+        var _a;
+        const retries = (_a = this.clientConfig.retries) !== null && _a !== void 0 ? _a : 0;
+        if (retries > 0) {
+            (0, axios_retry_1.default)(axios_1.default, {
+                retries,
+                retryDelay: axios_retry_1.default.exponentialDelay,
+                retryCondition: (error) => {
+                    var _a;
+                    if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
+                        return true;
+                    }
+                    return axios_retry_1.default.isNetworkOrIdempotentRequestError(error);
+                },
+            });
+        }
+    }
 }
 exports.ZenRows = ZenRows;

package/dist/queue.d.ts

@@ -0,0 +1,4 @@
+declare const queue: (concurrency: number) => {
+    enqueue: Function;
+};
+export { queue };

package/dist/queue.js

@@ -0,0 +1,37 @@
+"use strict";
+// TODO review disabled!!!!
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.queue = void 0;
+/* eslint-disable @typescript-eslint/ban-types */
+const queue = (concurrency) => {
+    let running = 0;
+    const tasks = [];
+    return {
+        enqueue: (givenTask, ...givenParams) => __awaiter(void 0, void 0, void 0, function* () {
+            tasks.push({ task: givenTask, params: givenParams });
+            if (running >= concurrency) {
+                return;
+            }
+            running += 1;
+            while (tasks.length > 0) {
+                const nextTask = tasks.shift();
+                if (nextTask) {
+                    const { task, params } = nextTask;
+                    // eslint-disable-next-line no-await-in-loop
+                    yield task(...params);
+                }
+            }
+            running -= 1;
+        }),
+    };
+};
+exports.queue = queue;

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "1.0.5";
+export declare const VERSION = "1.2.0";

package/dist/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '1.0.5';
+exports.VERSION = '1.2.0';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zenrows",
-  "version": "1.0.5",
+  "version": "1.2.0",
   "description": "ZenRows Node SDK",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -40,6 +40,8 @@
     "typescript": "^4.4.2"
   },
   "dependencies": {
-    "axios": "^0.21.4"
+    "axios": "^0.21.4",
+    "axios-retry": "^3.2.4",
+    "fastq": "^1.13.0"
   }
 }