proxy-rotator-js 1.0.9 → 1.1.1

package/README.md

@@ -100,6 +100,18 @@ const proxyRotator = new ProxyRotator(proxies, options);
     - assume_aliveness: Specifies whether to assume all proxies are alive when first added instead of 'new'. Default: false.
     - check_on_next: Specifies whether to check if proxies are alive when they are added to the queue. Default: false.
 
+## Testing your Proxies
+
+```javascript
+import ProxyRotator from 'proxy-rotator-js'
+
+let proxies = ['proxy1', 'proxy2', 'proxy3']
+
+let rotator = new ProxyRotator(proxies, options={})
+
+rotator.test()
+```
+
 ## Getting Started
 
 To use the ProxyRotator class in your JavaScript project, follow these steps:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "proxy-rotator-js",
-  "version": "1.0.9",
+  "version": "1.1.1",
   "type": "module",
   "description": "Proxy Rotator",
   "main": "index.js",
@@ -14,6 +14,7 @@
     "anymatch": "^3.1.3",
     "argparse": "^2.0.1",
     "assertion-error": "^1.1.0",
+    "axios": "^1.6.8",
     "balanced-match": "^1.0.2",
     "binary-extensions": "^2.2.0",
     "brace-expansion": "^2.0.1",

package/src/ProxyRotator.js

@@ -1,6 +1,7 @@
 import fs from 'fs';
 import Queue from './Queue.js';
 import Proxy from './Proxy.js';
+import makeRequestWithProxy from './utils/makeRequestWithProxy.js';
 
 class ProxyRotator {
     constructor(proxies, options={} ){
@@ -202,6 +203,37 @@ class ProxyRotator {
                 this.resurect(proxy);
     }
 
+    async test_proxies(){
+        // test if proxies are working
+        let proxies = this.pool.toArray()
+            .map(p=>p.proxy)
+            .map(p=> ({
+                host:p.split(':')[0],
+                port:p.split(':')[1]
+            }));
+        console.log('--- Testing Proxies ---');
+        let workingCount = 0;
+        let notWorkingCount = 0;
+        // make request with proxy
+        for (const proxy of proxies) {
+            console.log(`Testing proxy ${proxy.host}:${proxy.port}...`);    
+            const response = await makeRequestWithProxy(proxy);
+            if (response === proxy.host) {
+                console.log(`Proxy ${proxy.host}:${proxy.port} is working.`);
+                workingCount++;
+            } else {
+                console.log(`Proxy ${proxy.host}:${proxy.port} is not working.`);
+                notWorkingCount++;
+            }
+        }
+        console.log('--- Statistics ---');
+        console.log(`Total proxies: ${proxies.length}`);
+        console.log(`Working proxies: ${workingCount}`);
+        console.log(`Not working proxies: ${notWorkingCount}`);
+    }
+
+    test = this.test_proxies;
+
 }
 
 export default ProxyRotator 

package/src/utils/makeRequestWithProxy.js

@@ -0,0 +1,21 @@
+import axios from 'axios';
+// Description: Function to make an Axios request using a proxy
+
+// Function to make an Axios request using a proxy
+async function makeRequestWithProxy(proxy) {
+  try {
+    const response = await axios.get('https://api.ipify.org?format=json', {
+      proxy: {
+        host: proxy.host,
+        port: proxy.port,
+        protocol: 'http'
+      }
+    });
+    console.log(response.data);
+    return response.data;
+  } catch (error) {
+    return null;
+  }
+}
+
+export default makeRequestWithProxy;

package/test/test_proxies.js

@@ -0,0 +1,22 @@
+import ProxyRotator from '../index.js'
+import assert from 'assert';
+import chai from 'chai';
+const expect = chai.expect
+
+// slogan: when you like a little risk with you proxies
+// proxies to test with 
+let test_proxies = [ '139.59.1.14:8080', '94.45.74.60:8080', 
+    '161.35.70.249:3128', '217.182.170.224:80', '222.138.76.6:9002',
+    '218.252.206.89:80',  '18.214.66.210:80', '120.234.203.171:9002' ]
+
+// check if it is able to read from difrent files
+describe('testing testing proxies', () => {
+    // test if it can process a string point to a file
+    it('testing proxeis', () => {
+        // make rotator 
+        let rotator = new ProxyRotator(test_proxies);
+        rotator.test()
+        assert.deepEqual(test_proxies,rotator.getPool())
+    })
+})
+

package/test_proxies/node_modules/.package-lock.json

@@ -1,98 +0,0 @@
-{
-  "name": "test_proxies",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "node_modules/asynckit": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
-    },
-    "node_modules/axios": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz",
-      "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==",
-      "dependencies": {
-        "follow-redirects": "^1.15.0",
-        "form-data": "^4.0.0",
-        "proxy-from-env": "^1.1.0"
-      }
-    },
-    "node_modules/combined-stream": {
-      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
-      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dependencies": {
-        "delayed-stream": "~1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/delayed-stream": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "engines": {
-        "node": ">=0.4.0"
-      }
-    },
-    "node_modules/follow-redirects": {
-      "version": "1.15.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz",
-      "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==",
-      "funding": [
-        {
-          "type": "individual",
-          "url": "https://github.com/sponsors/RubenVerborgh"
-        }
-      ],
-      "engines": {
-        "node": ">=4.0"
-      },
-      "peerDependenciesMeta": {
-        "debug": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/form-data": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dependencies": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.8",
-        "mime-types": "^2.1.12"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
-    "node_modules/mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dependencies": {
-        "mime-db": "1.52.0"
-      },
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/proxy-from-env": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
-    }
-  }
-}

package/test_proxies/node_modules/asynckit/LICENSE

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2016 Alex Indigo
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/test_proxies/node_modules/asynckit/README.md

@@ -1,233 +0,0 @@
-# asynckit [![NPM Module](https://img.shields.io/npm/v/asynckit.svg?style=flat)](https://www.npmjs.com/package/asynckit)
-
-Minimal async jobs utility library, with streams support.
-
-[![PhantomJS Build](https://img.shields.io/travis/alexindigo/asynckit/v0.4.0.svg?label=browser&style=flat)](https://travis-ci.org/alexindigo/asynckit)
-[![Linux Build](https://img.shields.io/travis/alexindigo/asynckit/v0.4.0.svg?label=linux:0.12-6.x&style=flat)](https://travis-ci.org/alexindigo/asynckit)
-[![Windows Build](https://img.shields.io/appveyor/ci/alexindigo/asynckit/v0.4.0.svg?label=windows:0.12-6.x&style=flat)](https://ci.appveyor.com/project/alexindigo/asynckit)
-
-[![Coverage Status](https://img.shields.io/coveralls/alexindigo/asynckit/v0.4.0.svg?label=code+coverage&style=flat)](https://coveralls.io/github/alexindigo/asynckit?branch=master)
-[![Dependency Status](https://img.shields.io/david/alexindigo/asynckit/v0.4.0.svg?style=flat)](https://david-dm.org/alexindigo/asynckit)
-[![bitHound Overall Score](https://www.bithound.io/github/alexindigo/asynckit/badges/score.svg)](https://www.bithound.io/github/alexindigo/asynckit)
-
-<!-- [![Readme](https://img.shields.io/badge/readme-tested-brightgreen.svg?style=flat)](https://www.npmjs.com/package/reamde) -->
-
-AsyncKit provides harness for `parallel` and `serial` iterators over list of items represented by arrays or objects.
-Optionally it accepts abort function (should be synchronously return by iterator for each item), and terminates left over jobs upon an error event. For specific iteration order built-in (`ascending` and `descending`) and custom sort helpers also supported, via `asynckit.serialOrdered` method.
-
-It ensures async operations to keep behavior more stable and prevent `Maximum call stack size exceeded` errors, from sync iterators.
-
-| compression        |     size |
-| :----------------- | -------: |
-| asynckit.js        | 12.34 kB |
-| asynckit.min.js    |  4.11 kB |
-| asynckit.min.js.gz |  1.47 kB |
-
-
-## Install
-
-```sh
-$ npm install --save asynckit
-```
-
-## Examples
-
-### Parallel Jobs
-
-Runs iterator over provided array in parallel. Stores output in the `result` array,
-on the matching positions. In unlikely event of an error from one of the jobs,
-will terminate rest of the active jobs (if abort function is provided)
-and return error along with salvaged data to the main callback function.
-
-#### Input Array
-
-```javascript
-var parallel = require('asynckit').parallel
-  , assert   = require('assert')
-  ;
-
-var source         = [ 1, 1, 4, 16, 64, 32, 8, 2 ]
-  , expectedResult = [ 2, 2, 8, 32, 128, 64, 16, 4 ]
-  , expectedTarget = [ 1, 1, 2, 4, 8, 16, 32, 64 ]
-  , target         = []
-  ;
-
-parallel(source, asyncJob, function(err, result)
-{
-  assert.deepEqual(result, expectedResult);
-  assert.deepEqual(target, expectedTarget);
-});
-
-// async job accepts one element from the array
-// and a callback function
-function asyncJob(item, cb)
-{
-  // different delays (in ms) per item
-  var delay = item * 25;
-
-  // pretend different jobs take different time to finish
-  // and not in consequential order
-  var timeoutId = setTimeout(function() {
-    target.push(item);
-    cb(null, item * 2);
-  }, delay);
-
-  // allow to cancel "leftover" jobs upon error
-  // return function, invoking of which will abort this job
-  return clearTimeout.bind(null, timeoutId);
-}
-```
-
-More examples could be found in [test/test-parallel-array.js](test/test-parallel-array.js).
-
-#### Input Object
-
-Also it supports named jobs, listed via object.
-
-```javascript
-var parallel = require('asynckit/parallel')
-  , assert   = require('assert')
-  ;
-
-var source         = { first: 1, one: 1, four: 4, sixteen: 16, sixtyFour: 64, thirtyTwo: 32, eight: 8, two: 2 }
-  , expectedResult = { first: 2, one: 2, four: 8, sixteen: 32, sixtyFour: 128, thirtyTwo: 64, eight: 16, two: 4 }
-  , expectedTarget = [ 1, 1, 2, 4, 8, 16, 32, 64 ]
-  , expectedKeys   = [ 'first', 'one', 'two', 'four', 'eight', 'sixteen', 'thirtyTwo', 'sixtyFour' ]
-  , target         = []
-  , keys           = []
-  ;
-
-parallel(source, asyncJob, function(err, result)
-{
-  assert.deepEqual(result, expectedResult);
-  assert.deepEqual(target, expectedTarget);
-  assert.deepEqual(keys, expectedKeys);
-});
-
-// supports full value, key, callback (shortcut) interface
-function asyncJob(item, key, cb)
-{
-  // different delays (in ms) per item
-  var delay = item * 25;
-
-  // pretend different jobs take different time to finish
-  // and not in consequential order
-  var timeoutId = setTimeout(function() {
-    keys.push(key);
-    target.push(item);
-    cb(null, item * 2);
-  }, delay);
-
-  // allow to cancel "leftover" jobs upon error
-  // return function, invoking of which will abort this job
-  return clearTimeout.bind(null, timeoutId);
-}
-```
-
-More examples could be found in [test/test-parallel-object.js](test/test-parallel-object.js).
-
-### Serial Jobs
-
-Runs iterator over provided array sequentially. Stores output in the `result` array,
-on the matching positions. In unlikely event of an error from one of the jobs,
-will not proceed to the rest of the items in the list
-and return error along with salvaged data to the main callback function.
-
-#### Input Array
-
-```javascript
-var serial = require('asynckit/serial')
-  , assert = require('assert')
-  ;
-
-var source         = [ 1, 1, 4, 16, 64, 32, 8, 2 ]
-  , expectedResult = [ 2, 2, 8, 32, 128, 64, 16, 4 ]
-  , expectedTarget = [ 0, 1, 2, 3, 4, 5, 6, 7 ]
-  , target         = []
-  ;
-
-serial(source, asyncJob, function(err, result)
-{
-  assert.deepEqual(result, expectedResult);
-  assert.deepEqual(target, expectedTarget);
-});
-
-// extended interface (item, key, callback)
-// also supported for arrays
-function asyncJob(item, key, cb)
-{
-  target.push(key);
-
-  // it will be automatically made async
-  // even it iterator "returns" in the same event loop
-  cb(null, item * 2);
-}
-```
-
-More examples could be found in [test/test-serial-array.js](test/test-serial-array.js).
-
-#### Input Object
-
-Also it supports named jobs, listed via object.
-
-```javascript
-var serial = require('asynckit').serial
-  , assert = require('assert')
-  ;
-
-var source         = [ 1, 1, 4, 16, 64, 32, 8, 2 ]
-  , expectedResult = [ 2, 2, 8, 32, 128, 64, 16, 4 ]
-  , expectedTarget = [ 0, 1, 2, 3, 4, 5, 6, 7 ]
-  , target         = []
-  ;
-
-var source         = { first: 1, one: 1, four: 4, sixteen: 16, sixtyFour: 64, thirtyTwo: 32, eight: 8, two: 2 }
-  , expectedResult = { first: 2, one: 2, four: 8, sixteen: 32, sixtyFour: 128, thirtyTwo: 64, eight: 16, two: 4 }
-  , expectedTarget = [ 1, 1, 4, 16, 64, 32, 8, 2 ]
-  , target         = []
-  ;
-
-
-serial(source, asyncJob, function(err, result)
-{
-  assert.deepEqual(result, expectedResult);
-  assert.deepEqual(target, expectedTarget);
-});
-
-// shortcut interface (item, callback)
-// works for object as well as for the arrays
-function asyncJob(item, cb)
-{
-  target.push(item);
-
-  // it will be automatically made async
-  // even it iterator "returns" in the same event loop
-  cb(null, item * 2);
-}
-```
-
-More examples could be found in [test/test-serial-object.js](test/test-serial-object.js).
-
-_Note: Since _object_ is an _unordered_ collection of properties,
-it may produce unexpected results with sequential iterations.
-Whenever order of the jobs' execution is important please use `serialOrdered` method._
-
-### Ordered Serial Iterations
-
-TBD
-
-For example [compare-property](compare-property) package.
-
-### Streaming interface
-
-TBD
-
-## Want to Know More?
-
-More examples can be found in [test folder](test/).
-
-Or open an [issue](https://github.com/alexindigo/asynckit/issues) with questions and/or suggestions.
-
-## License
-
-AsyncKit is licensed under the MIT license.

package/test_proxies/node_modules/asynckit/bench.js

@@ -1,76 +0,0 @@
-/* eslint no-console: "off" */
-
-var asynckit = require('./')
-  , async    = require('async')
-  , assert   = require('assert')
-  , expected = 0
-  ;
-
-var Benchmark = require('benchmark');
-var suite = new Benchmark.Suite;
-
-var source = [];
-for (var z = 1; z < 100; z++)
-{
-  source.push(z);
-  expected += z;
-}
-
-suite
-// add tests
-
-.add('async.map', function(deferred)
-{
-  var total = 0;
-
-  async.map(source,
-  function(i, cb)
-  {
-    setImmediate(function()
-    {
-      total += i;
-      cb(null, total);
-    });
-  },
-  function(err, result)
-  {
-    assert.ifError(err);
-    assert.equal(result[result.length - 1], expected);
-    deferred.resolve();
-  });
-}, {'defer': true})
-
-
-.add('asynckit.parallel', function(deferred)
-{
-  var total = 0;
-
-  asynckit.parallel(source,
-  function(i, cb)
-  {
-    setImmediate(function()
-    {
-      total += i;
-      cb(null, total);
-    });
-  },
-  function(err, result)
-  {
-    assert.ifError(err);
-    assert.equal(result[result.length - 1], expected);
-    deferred.resolve();
-  });
-}, {'defer': true})
-
-
-// add listeners
-.on('cycle', function(ev)
-{
-  console.log(String(ev.target));
-})
-.on('complete', function()
-{
-  console.log('Fastest is ' + this.filter('fastest').map('name'));
-})
-// run async
-.run({ 'async': true });

package/test_proxies/node_modules/asynckit/index.js

@@ -1,6 +0,0 @@
-module.exports =
-{
-  parallel      : require('./parallel.js'),
-  serial        : require('./serial.js'),
-  serialOrdered : require('./serialOrdered.js')
-};

package/test_proxies/node_modules/asynckit/lib/abort.js

@@ -1,29 +0,0 @@
-// API
-module.exports = abort;
-
-/**
- * Aborts leftover active jobs
- *
- * @param {object} state - current state object
- */
-function abort(state)
-{
-  Object.keys(state.jobs).forEach(clean.bind(state));
-
-  // reset leftover jobs
-  state.jobs = {};
-}
-
-/**
- * Cleans up leftover job by invoking abort function for the provided job id
- *
- * @this  state
- * @param {string|number} key - job id to abort
- */
-function clean(key)
-{
-  if (typeof this.jobs[key] == 'function')
-  {
-    this.jobs[key]();
-  }
-}

package/test_proxies/node_modules/asynckit/lib/async.js

@@ -1,34 +0,0 @@
-var defer = require('./defer.js');
-
-// API
-module.exports = async;
-
-/**
- * Runs provided callback asynchronously
- * even if callback itself is not
- *
- * @param   {function} callback - callback to invoke
- * @returns {function} - augmented callback
- */
-function async(callback)
-{
-  var isAsync = false;
-
-  // check if async happened
-  defer(function() { isAsync = true; });
-
-  return function async_callback(err, result)
-  {
-    if (isAsync)
-    {
-      callback(err, result);
-    }
-    else
-    {
-      defer(function nextTick_callback()
-      {
-        callback(err, result);
-      });
-    }
-  };
-}

package/test_proxies/node_modules/asynckit/lib/defer.js

@@ -1,26 +0,0 @@
-module.exports = defer;
-
-/**
- * Runs provided function on next iteration of the event loop
- *
- * @param {function} fn - function to run
- */
-function defer(fn)
-{
-  var nextTick = typeof setImmediate == 'function'
-    ? setImmediate
-    : (
-      typeof process == 'object' && typeof process.nextTick == 'function'
-      ? process.nextTick
-      : null
-    );
-
-  if (nextTick)
-  {
-    nextTick(fn);
-  }
-  else
-  {
-    setTimeout(fn, 0);
-  }
-}