proxy-rotator-js 1.0.8 → 1.0.9

package/README.md

@@ -77,6 +77,29 @@ remove(proxy) // Removes one or more proxies from the pool.
 
     graveyard: Stores proxies that are currently dead or inactive.
 
+## Options
+```javascript
+const proxies = ['proxy1.example.com', 'proxy2.example.com'];
+const options = {
+    returnAs: 'object',
+    revive_timer: 1000 * 60 * 30,
+    shuffle: true,
+    protocol: 'http',
+    assume_aliveness: true,
+    check_on_next: true
+};
+
+const proxyRotator = new ProxyRotator(proxies, options);
+```
+    The following options can be passed to customize the behavior of the ProxyRotator:
+
+    - returnAs: Specifies the return type of proxies. Can be either 'string' or 'object'. Default: 'string'.
+    - revive_timer: Specifies the duration in milliseconds before a dead proxy is revived. Default: 1000 * 60 * 30 (30 minutes).
+    - protocol: Specifies a protocol for all proxies. Default: null.
+    - shuffle: Specifies whether to shuffle the proxies before adding them to the queue. Default: false.
+    - assume_aliveness: Specifies whether to assume all proxies are alive when first added instead of 'new'. Default: false.
+    - check_on_next: Specifies whether to check if proxies are alive when they are added to the queue. Default: false.
+
 ## Getting Started
 
 To use the ProxyRotator class in your JavaScript project, follow these steps:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "proxy-rotator-js",
-  "version": "1.0.8",
+  "version": "1.0.9",
   "type": "module",
   "description": "Proxy Rotator",
   "main": "index.js",

package/src/ProxyRotator.js

@@ -16,7 +16,7 @@ class ProxyRotator {
         // assume a a protocol for all proxies
         this.protocol = protocol ?? null;
         // shuffle the proxies before adding them to the queue
-        this.shuffle = shuffle ?? true;
+        this.shuffle = shuffle ?? false;
         // assume all proxies are alive when first added instead of 'new'
         this.assume_aliveness = assume_aliveness ?? false;
         // check if proxies are alive when they are added to the queue
@@ -32,6 +32,12 @@ class ProxyRotator {
             // add proxies to queue
             proxies.forEach( p => this._add(p) );
         }
+        // shuffle proxies
+        if(this.shuffle){
+            let shuffled = this._shuffleArray(this.pool.toArray());
+            this.pool = new Queue();
+            shuffled.forEach( p => this.pool.enqueue(p) );
+        }
     }
 
     getGraveyard(){ return this.graveyard.map(p => p.proxy) }

package/test/options.js

@@ -0,0 +1,32 @@
+import ProxyRotator from '../index.js'
+import assert from 'assert';
+import chai from 'chai';
+const expect = chai.expect
+
+// slogan: when you like a little risk with you proxies
+// proxies to test with 
+let test_proxies = [ '139.59.1.14:8080', '94.45.74.60:8080', 
+    '161.35.70.249:3128', '217.182.170.224:80', '222.138.76.6:9002',
+    '218.252.206.89:80',  '18.214.66.210:80', '120.234.203.171:9002' ]
+
+// check shuffle option
+describe('shuffle option', () => {
+    it('shuffle proxies when true', () => {
+        // make rotator with shuffle option set to true
+        let rotator = new ProxyRotator(test_proxies, { shuffle: true });
+        // get pool
+        let pool = rotator.getPool();
+        // check if the proxies are shuffled
+        assert.notDeepEqual(test_proxies, pool);
+    });
+
+    it('do not shuffle proxies when false', () => {
+        // make rotator with shuffle option set to false
+        let rotator = new ProxyRotator(test_proxies, { shuffle: false });
+        // get pool
+        let pool = rotator.getPool();
+        // check if the proxies are not shuffled
+        assert.deepEqual(test_proxies, pool);
+    });
+});
+

package/test_proxies/node_modules/.package-lock.json

@@ -0,0 +1,98 @@
+{
+  "name": "test_proxies",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "node_modules/axios": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz",
+      "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==",
+      "dependencies": {
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
+      }
+    },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/follow-redirects": {
+      "version": "1.15.3",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz",
+      "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    }
+  }
+}

package/test_proxies/node_modules/asynckit/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Alex Indigo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/test_proxies/node_modules/asynckit/README.md

@@ -0,0 +1,233 @@
+# asynckit [![NPM Module](https://img.shields.io/npm/v/asynckit.svg?style=flat)](https://www.npmjs.com/package/asynckit)
+
+Minimal async jobs utility library, with streams support.
+
+[![PhantomJS Build](https://img.shields.io/travis/alexindigo/asynckit/v0.4.0.svg?label=browser&style=flat)](https://travis-ci.org/alexindigo/asynckit)
+[![Linux Build](https://img.shields.io/travis/alexindigo/asynckit/v0.4.0.svg?label=linux:0.12-6.x&style=flat)](https://travis-ci.org/alexindigo/asynckit)
+[![Windows Build](https://img.shields.io/appveyor/ci/alexindigo/asynckit/v0.4.0.svg?label=windows:0.12-6.x&style=flat)](https://ci.appveyor.com/project/alexindigo/asynckit)
+
+[![Coverage Status](https://img.shields.io/coveralls/alexindigo/asynckit/v0.4.0.svg?label=code+coverage&style=flat)](https://coveralls.io/github/alexindigo/asynckit?branch=master)
+[![Dependency Status](https://img.shields.io/david/alexindigo/asynckit/v0.4.0.svg?style=flat)](https://david-dm.org/alexindigo/asynckit)
+[![bitHound Overall Score](https://www.bithound.io/github/alexindigo/asynckit/badges/score.svg)](https://www.bithound.io/github/alexindigo/asynckit)
+
+<!-- [![Readme](https://img.shields.io/badge/readme-tested-brightgreen.svg?style=flat)](https://www.npmjs.com/package/reamde) -->
+
+AsyncKit provides harness for `parallel` and `serial` iterators over list of items represented by arrays or objects.
+Optionally it accepts abort function (should be synchronously return by iterator for each item), and terminates left over jobs upon an error event. For specific iteration order built-in (`ascending` and `descending`) and custom sort helpers also supported, via `asynckit.serialOrdered` method.
+
+It ensures async operations to keep behavior more stable and prevent `Maximum call stack size exceeded` errors, from sync iterators.
+
+| compression        |     size |
+| :----------------- | -------: |
+| asynckit.js        | 12.34 kB |
+| asynckit.min.js    |  4.11 kB |
+| asynckit.min.js.gz |  1.47 kB |
+
+
+## Install
+
+```sh
+$ npm install --save asynckit
+```
+
+## Examples
+
+### Parallel Jobs
+
+Runs iterator over provided array in parallel. Stores output in the `result` array,
+on the matching positions. In unlikely event of an error from one of the jobs,
+will terminate rest of the active jobs (if abort function is provided)
+and return error along with salvaged data to the main callback function.
+
+#### Input Array
+
+```javascript
+var parallel = require('asynckit').parallel
+  , assert   = require('assert')
+  ;
+
+var source         = [ 1, 1, 4, 16, 64, 32, 8, 2 ]
+  , expectedResult = [ 2, 2, 8, 32, 128, 64, 16, 4 ]
+  , expectedTarget = [ 1, 1, 2, 4, 8, 16, 32, 64 ]
+  , target         = []
+  ;
+
+parallel(source, asyncJob, function(err, result)
+{
+  assert.deepEqual(result, expectedResult);
+  assert.deepEqual(target, expectedTarget);
+});
+
+// async job accepts one element from the array
+// and a callback function
+function asyncJob(item, cb)
+{
+  // different delays (in ms) per item
+  var delay = item * 25;
+
+  // pretend different jobs take different time to finish
+  // and not in consequential order
+  var timeoutId = setTimeout(function() {
+    target.push(item);
+    cb(null, item * 2);
+  }, delay);
+
+  // allow to cancel "leftover" jobs upon error
+  // return function, invoking of which will abort this job
+  return clearTimeout.bind(null, timeoutId);
+}
+```
+
+More examples could be found in [test/test-parallel-array.js](test/test-parallel-array.js).
+
+#### Input Object
+
+Also it supports named jobs, listed via object.
+
+```javascript
+var parallel = require('asynckit/parallel')
+  , assert   = require('assert')
+  ;
+
+var source         = { first: 1, one: 1, four: 4, sixteen: 16, sixtyFour: 64, thirtyTwo: 32, eight: 8, two: 2 }
+  , expectedResult = { first: 2, one: 2, four: 8, sixteen: 32, sixtyFour: 128, thirtyTwo: 64, eight: 16, two: 4 }
+  , expectedTarget = [ 1, 1, 2, 4, 8, 16, 32, 64 ]
+  , expectedKeys   = [ 'first', 'one', 'two', 'four', 'eight', 'sixteen', 'thirtyTwo', 'sixtyFour' ]
+  , target         = []
+  , keys           = []
+  ;
+
+parallel(source, asyncJob, function(err, result)
+{
+  assert.deepEqual(result, expectedResult);
+  assert.deepEqual(target, expectedTarget);
+  assert.deepEqual(keys, expectedKeys);
+});
+
+// supports full value, key, callback (shortcut) interface
+function asyncJob(item, key, cb)
+{
+  // different delays (in ms) per item
+  var delay = item * 25;
+
+  // pretend different jobs take different time to finish
+  // and not in consequential order
+  var timeoutId = setTimeout(function() {
+    keys.push(key);
+    target.push(item);
+    cb(null, item * 2);
+  }, delay);
+
+  // allow to cancel "leftover" jobs upon error
+  // return function, invoking of which will abort this job
+  return clearTimeout.bind(null, timeoutId);
+}
+```
+
+More examples could be found in [test/test-parallel-object.js](test/test-parallel-object.js).
+
+### Serial Jobs
+
+Runs iterator over provided array sequentially. Stores output in the `result` array,
+on the matching positions. In unlikely event of an error from one of the jobs,
+will not proceed to the rest of the items in the list
+and return error along with salvaged data to the main callback function.
+
+#### Input Array
+
+```javascript
+var serial = require('asynckit/serial')
+  , assert = require('assert')
+  ;
+
+var source         = [ 1, 1, 4, 16, 64, 32, 8, 2 ]
+  , expectedResult = [ 2, 2, 8, 32, 128, 64, 16, 4 ]
+  , expectedTarget = [ 0, 1, 2, 3, 4, 5, 6, 7 ]
+  , target         = []
+  ;
+
+serial(source, asyncJob, function(err, result)
+{
+  assert.deepEqual(result, expectedResult);
+  assert.deepEqual(target, expectedTarget);
+});
+
+// extended interface (item, key, callback)
+// also supported for arrays
+function asyncJob(item, key, cb)
+{
+  target.push(key);
+
+  // it will be automatically made async
+  // even it iterator "returns" in the same event loop
+  cb(null, item * 2);
+}
+```
+
+More examples could be found in [test/test-serial-array.js](test/test-serial-array.js).
+
+#### Input Object
+
+Also it supports named jobs, listed via object.
+
+```javascript
+var serial = require('asynckit').serial
+  , assert = require('assert')
+  ;
+
+var source         = [ 1, 1, 4, 16, 64, 32, 8, 2 ]
+  , expectedResult = [ 2, 2, 8, 32, 128, 64, 16, 4 ]
+  , expectedTarget = [ 0, 1, 2, 3, 4, 5, 6, 7 ]
+  , target         = []
+  ;
+
+var source         = { first: 1, one: 1, four: 4, sixteen: 16, sixtyFour: 64, thirtyTwo: 32, eight: 8, two: 2 }
+  , expectedResult = { first: 2, one: 2, four: 8, sixteen: 32, sixtyFour: 128, thirtyTwo: 64, eight: 16, two: 4 }
+  , expectedTarget = [ 1, 1, 4, 16, 64, 32, 8, 2 ]
+  , target         = []
+  ;
+
+
+serial(source, asyncJob, function(err, result)
+{
+  assert.deepEqual(result, expectedResult);
+  assert.deepEqual(target, expectedTarget);
+});
+
+// shortcut interface (item, callback)
+// works for object as well as for the arrays
+function asyncJob(item, cb)
+{
+  target.push(item);
+
+  // it will be automatically made async
+  // even it iterator "returns" in the same event loop
+  cb(null, item * 2);
+}
+```
+
+More examples could be found in [test/test-serial-object.js](test/test-serial-object.js).
+
+_Note: Since _object_ is an _unordered_ collection of properties,
+it may produce unexpected results with sequential iterations.
+Whenever order of the jobs' execution is important please use `serialOrdered` method._
+
+### Ordered Serial Iterations
+
+TBD
+
+For example [compare-property](compare-property) package.
+
+### Streaming interface
+
+TBD
+
+## Want to Know More?
+
+More examples can be found in [test folder](test/).
+
+Or open an [issue](https://github.com/alexindigo/asynckit/issues) with questions and/or suggestions.
+
+## License
+
+AsyncKit is licensed under the MIT license.

package/test_proxies/node_modules/asynckit/bench.js

@@ -0,0 +1,76 @@
+/* eslint no-console: "off" */
+
+var asynckit = require('./')
+  , async    = require('async')
+  , assert   = require('assert')
+  , expected = 0
+  ;
+
+var Benchmark = require('benchmark');
+var suite = new Benchmark.Suite;
+
+var source = [];
+for (var z = 1; z < 100; z++)
+{
+  source.push(z);
+  expected += z;
+}
+
+suite
+// add tests
+
+.add('async.map', function(deferred)
+{
+  var total = 0;
+
+  async.map(source,
+  function(i, cb)
+  {
+    setImmediate(function()
+    {
+      total += i;
+      cb(null, total);
+    });
+  },
+  function(err, result)
+  {
+    assert.ifError(err);
+    assert.equal(result[result.length - 1], expected);
+    deferred.resolve();
+  });
+}, {'defer': true})
+
+
+.add('asynckit.parallel', function(deferred)
+{
+  var total = 0;
+
+  asynckit.parallel(source,
+  function(i, cb)
+  {
+    setImmediate(function()
+    {
+      total += i;
+      cb(null, total);
+    });
+  },
+  function(err, result)
+  {
+    assert.ifError(err);
+    assert.equal(result[result.length - 1], expected);
+    deferred.resolve();
+  });
+}, {'defer': true})
+
+
+// add listeners
+.on('cycle', function(ev)
+{
+  console.log(String(ev.target));
+})
+.on('complete', function()
+{
+  console.log('Fastest is ' + this.filter('fastest').map('name'));
+})
+// run async
+.run({ 'async': true });

package/test_proxies/node_modules/asynckit/index.js

@@ -0,0 +1,6 @@
+module.exports =
+{
+  parallel      : require('./parallel.js'),
+  serial        : require('./serial.js'),
+  serialOrdered : require('./serialOrdered.js')
+};

package/test_proxies/node_modules/asynckit/lib/abort.js

@@ -0,0 +1,29 @@
+// API
+module.exports = abort;
+
+/**
+ * Aborts leftover active jobs
+ *
+ * @param {object} state - current state object
+ */
+function abort(state)
+{
+  Object.keys(state.jobs).forEach(clean.bind(state));
+
+  // reset leftover jobs
+  state.jobs = {};
+}
+
+/**
+ * Cleans up leftover job by invoking abort function for the provided job id
+ *
+ * @this  state
+ * @param {string|number} key - job id to abort
+ */
+function clean(key)
+{
+  if (typeof this.jobs[key] == 'function')
+  {
+    this.jobs[key]();
+  }
+}

package/test_proxies/node_modules/asynckit/lib/async.js

@@ -0,0 +1,34 @@
+var defer = require('./defer.js');
+
+// API
+module.exports = async;
+
+/**
+ * Runs provided callback asynchronously
+ * even if callback itself is not
+ *
+ * @param   {function} callback - callback to invoke
+ * @returns {function} - augmented callback
+ */
+function async(callback)
+{
+  var isAsync = false;
+
+  // check if async happened
+  defer(function() { isAsync = true; });
+
+  return function async_callback(err, result)
+  {
+    if (isAsync)
+    {
+      callback(err, result);
+    }
+    else
+    {
+      defer(function nextTick_callback()
+      {
+        callback(err, result);
+      });
+    }
+  };
+}

package/test_proxies/node_modules/asynckit/lib/defer.js

@@ -0,0 +1,26 @@
+module.exports = defer;
+
+/**
+ * Runs provided function on next iteration of the event loop
+ *
+ * @param {function} fn - function to run
+ */
+function defer(fn)
+{
+  var nextTick = typeof setImmediate == 'function'
+    ? setImmediate
+    : (
+      typeof process == 'object' && typeof process.nextTick == 'function'
+      ? process.nextTick
+      : null
+    );
+
+  if (nextTick)
+  {
+    nextTick(fn);
+  }
+  else
+  {
+    setTimeout(fn, 0);
+  }
+}