proxy-rotator-js 1.1.1 → 1.2.1

package/assets/http_proxies_with_newlines.txt

@@ -1,8 +0,0 @@
-139.59.1.14:8080
-94.45.74.60:8080
-161.35.70.249:3128
-217.182.170.224:80
-222.138.76.6:9002
-218.252.206.89:80
-18.214.66.210:80
-120.234.203.171:9002

package/assets/http_proxies_with_spaces.txt

@@ -1 +0,0 @@
-139.59.1.14:8080 94.45.74.60:8080 161.35.70.249:3128 217.182.170.224:80 222.138.76.6:9002 218.252.206.89:80 18.214.66.210:80 120.234.203.171:9002

package/index.js

@@ -1,2 +0,0 @@
-import ProxyRotator from './src/ProxyRotator.js'
-export default ProxyRotator

package/src/ProxyRotator.js

@@ -1,239 +0,0 @@
-import fs from 'fs';
-import Queue from './Queue.js';
-import Proxy from './Proxy.js';
-import makeRequestWithProxy from './utils/makeRequestWithProxy.js';
-
-class ProxyRotator {
-    constructor(proxies, options={} ){
-        this.pool = new Queue();
-        this.graveyard = [];
-        // handle options
-        let { returnAs, revive_timer, shuffle, protocol, assume_aliveness, check_on_next } = options;
-        // how long to wait before reviving a dead proxy
-        // default: 30 minutes
-        this.revive_timer = revive_timer ?? 1000 * 60 * 30;
-        // return type of proxy, either 'string' or 'object'
-        this.returnAs = returnAs? this._handleReturnAsInput(returnAs) : 'string';
-        // assume a a protocol for all proxies
-        this.protocol = protocol ?? null;
-        // shuffle the proxies before adding them to the queue
-        this.shuffle = shuffle ?? false;
-        // assume all proxies are alive when first added instead of 'new'
-        this.assume_aliveness = assume_aliveness ?? false;
-        // check if proxies are alive when they are added to the queue
-        this.check_on_next = check_on_next ?? false;
-        // examine proxies passed
-        // check if it is a file path
-        if( typeof proxies === 'string' ){
-            // parse file
-            proxies = this._parseFile(proxies);
-            // add proxies to queue
-            proxies.forEach( p => this._add(p) );
-        }else if( this._isArray(proxies) ){
-            // add proxies to queue
-            proxies.forEach( p => this._add(p) );
-        }
-        // shuffle proxies
-        if(this.shuffle){
-            let shuffled = this._shuffleArray(this.pool.toArray());
-            this.pool = new Queue();
-            shuffled.forEach( p => this.pool.enqueue(p) );
-        }
-    }
-
-    getGraveyard(){ return this.graveyard.map(p => p.proxy) }
-
-    getGraveyardSize(){ return this.graveyard.length }
-
-    getPool(){ return this.pool.toArray().map(p => p.proxy) }
-
-    getPoolSize(){ return this.pool.size }
-
-    add(proxies){ // add proxy to queue
-        if(this._isArray(proxies)) // if passed an array
-            for(let proxy of proxies) this._add(proxy);
-        else // single file
-            this._add(proxies);
-    }
-
-    _add(proxy){ // add proxy to queue
-        let p = new Proxy(proxy, this.protocol, this.assume_aliveness);
-        this.pool.enqueue(p);
-    }
-
-    remove(proxy){ // remove proxy from queue
-        if(this._isArray(proxy)) // if passed an array
-            for(let p of proxy) this._remove(p);
-        else // single file
-            this._remove(proxy);
-    }
-
-    _remove(proxy){ // remove proxy from queue
-        this.pool.toArray().forEach( (p,i) => {
-            if(p.equals(proxy)) this.pool.remove(i);
-        });
-    }
-
-    getAlive(){ // get a random alive proxy
-        let proxies = this.pool.toArray();
-        for( let proxy of proxies )
-            if(proxy.isAlive()) return proxy.proxy;
-    }
-
-    setAlive(proxy){ // set a proxy to alive
-        let proxies = this.pool.toArray()
-        for( let p of proxies ) // if proxy is in queue
-            if(p.equals(proxy)) return p.setAlive();
-        // if proxy is in graveyard
-        for( let p of this.graveyard )
-            if(p.equals(proxy)) return this.resurect(p);
-    }
-
-    resurect( proxy ){
-        // get proxy from graveyard
-        let p = this.graveyard.find( p => p.equals(proxy) );
-        // if proxy is not in graveyard
-        if(!p) return;
-        // remove from graveyard
-        this.graveyard = this.graveyard.filter( p => !p.equals(proxy) );
-        // set as new
-        p.setNew();
-        // add to queue
-        this.pool.enqueue(p);
-    }
-
-    setDead(proxy){ // set a proxy to dead
-        this.pool.toArray().forEach( (p,i) => {
-            if(p.equals(proxy)){
-                p.setDead(); // set to dead
-                // remove from queue
-                this.pool.remove(i);
-                // add to graveyard
-                this.graveyard.push(p);
-            }
-        })
-        //  revive proxy after revive_timer
-        setTimeout( () => this.resurect(proxy), this.revive_timer );
-    }
-
-    kill(proxy){ // kill a proxy
-        return this.setDead(proxy);
-    }
-
-    next(options={}){
-        let { returnAs } = options;
-        // resurect a proxy from the graveyard
-        if(this.check_on_next) _resurection();
-        // if there are no proxies in the pool
-        if(this.pool.size === 0) return null ;
-        // remove from front 
-        let proxy = this.pool.dequeue();
-        // add to back
-        this.pool.enqueue(proxy);
-        // if parameter is passed
-        if(returnAs) returnAs = this._handleReturnAsInput(returnAs)
-        // else use default
-        else returnAs = this.returnAs;
-        // retun proxy as string or object
-        if(returnAs === 'string') return proxy.toString();
-        else if(returnAs === 'object') return proxy.obj()
-    }
-
-    /* Randomize array in-place using Durstenfeld shuffle algorithm */
-    _shuffleArray(array) {
-        for (var i = array.length - 1; i > 0; i--) {
-            var j = Math.floor(Math.random() * (i + 1));
-            var temp = array[i];
-            array[i] = array[j];
-            array[j] = temp;
-        }
-        return array;
-    }
-
-    _handleReturnAsInput(proxyType=null){
-        if(typeof proxyType === 'string'){
-            proxyType = proxyType.toLowerCase();
-            if(proxyType === 'str') proxyType = 'string';
-            if(proxyType === 'obj') proxyType = 'object';
-            if(proxyType !== 'string' && proxyType !== 'object'){
-                console.error('proxyType must be either "string" or "object"');
-                proxyType = null
-            }
-            return proxyType;
-        }else
-            return null;
-    }
-    
-    _parseFile(filename) {
-        // read file
-        let str = fs.readFileSync(filename, 'utf8');
-        // this function is able to handle multiples types of files
-        let strList = str.split('\n')
-        // remove lines that are empty
-        strList = strList.filter(s=>s.length>0);
-        // if strList is has only one element
-        if(strList.length === 1)
-            strList = str.split(' ')
-        // if strList is has only one check to separate by comma
-        if(strList.length === 1)
-            strList = str.split(',')
-        // remove all the commas from the strings
-        strList = strList.map(s=>s.replace(',',''))
-        // parse list of strings into 
-        strList = strList
-            .map(s=>s.trim())
-            .filter(s=>s.length>0);
-        // return proxies
-        return strList;
-    }
-
-
-    _isArray(arrayValue){
-        return ( arrayValue && 
-            (typeof arrayValue === 'object') && 
-            (arrayValue.constructor === Array) );
-    }
-
-    _resurection(){
-        // look for proxies that have been 
-        // dead for a long than the resurection time
-        // and revive them
-        for( let proxy of this.graveyard )
-            if( proxy.isDead() && proxy.timeSinceStatusChange >= this.revive_timer )
-                this.resurect(proxy);
-    }
-
-    async test_proxies(){
-        // test if proxies are working
-        let proxies = this.pool.toArray()
-            .map(p=>p.proxy)
-            .map(p=> ({
-                host:p.split(':')[0],
-                port:p.split(':')[1]
-            }));
-        console.log('--- Testing Proxies ---');
-        let workingCount = 0;
-        let notWorkingCount = 0;
-        // make request with proxy
-        for (const proxy of proxies) {
-            console.log(`Testing proxy ${proxy.host}:${proxy.port}...`);    
-            const response = await makeRequestWithProxy(proxy);
-            if (response === proxy.host) {
-                console.log(`Proxy ${proxy.host}:${proxy.port} is working.`);
-                workingCount++;
-            } else {
-                console.log(`Proxy ${proxy.host}:${proxy.port} is not working.`);
-                notWorkingCount++;
-            }
-        }
-        console.log('--- Statistics ---');
-        console.log(`Total proxies: ${proxies.length}`);
-        console.log(`Working proxies: ${workingCount}`);
-        console.log(`Not working proxies: ${notWorkingCount}`);
-    }
-
-    test = this.test_proxies;
-
-}
-
-export default ProxyRotator 

package/src/Queue.js

@@ -1,61 +0,0 @@
-class Queue {
-    constructor() {
-        this.items = {}
-        this.frontIndex = 0
-        this.backIndex = 0
-    }
-    
-    enqueue(item) {
-        this.items[this.backIndex] = item
-        this.backIndex++
-        return item + ' inserted'
-    }
-
-    dequeue() {
-        const item = this.items[this.frontIndex]
-        delete this.items[this.frontIndex]
-        this.frontIndex++
-        return item
-    }
-    
-    // remove value while maintaining order
-    remove( index ) {
-        if (index < 0 || index >= this.backIndex) {
-            return 'Invalid index'
-        }
-        const item = this.items[index]
-        for (let i = index; i < this.backIndex; i++) {
-            this.items[i] = this.items[i + 1]
-        }
-        delete this.items[this.backIndex - 1]
-        this.backIndex--
-        return item
-    }
-
-    peek() {
-        return this.items[this.frontIndex]
-    }
-
-    printQueue() {
-        return this.items;
-    }
-
-    // return array of items in the order they were added
-    toArray() {
-        let arr = [];
-        for(let i = this.frontIndex; i < this.backIndex; i++)
-            arr.push(this.items[i]);
-        return arr;
-    }
-
-    // return object of items in the order they were added
-    toObject() {
-        let obj = {};
-        for(let i = this.frontIndex; i < this.backIndex; i++)
-            obj[i] = this.items[i];
-        return obj;
-    }
-    
-}
-
-export default Queue;

package/src/utils/makeRequestWithProxy.js

@@ -1,21 +0,0 @@
-import axios from 'axios';
-// Description: Function to make an Axios request using a proxy
-
-// Function to make an Axios request using a proxy
-async function makeRequestWithProxy(proxy) {
-  try {
-    const response = await axios.get('https://api.ipify.org?format=json', {
-      proxy: {
-        host: proxy.host,
-        port: proxy.port,
-        protocol: 'http'
-      }
-    });
-    console.log(response.data);
-    return response.data;
-  } catch (error) {
-    return null;
-  }
-}
-
-export default makeRequestWithProxy;

package/test/options.js

@@ -1,32 +0,0 @@
-import ProxyRotator from '../index.js'
-import assert from 'assert';
-import chai from 'chai';
-const expect = chai.expect
-
-// slogan: when you like a little risk with you proxies
-// proxies to test with 
-let test_proxies = [ '139.59.1.14:8080', '94.45.74.60:8080', 
-    '161.35.70.249:3128', '217.182.170.224:80', '222.138.76.6:9002',
-    '218.252.206.89:80',  '18.214.66.210:80', '120.234.203.171:9002' ]
-
-// check shuffle option
-describe('shuffle option', () => {
-    it('shuffle proxies when true', () => {
-        // make rotator with shuffle option set to true
-        let rotator = new ProxyRotator(test_proxies, { shuffle: true });
-        // get pool
-        let pool = rotator.getPool();
-        // check if the proxies are shuffled
-        assert.notDeepEqual(test_proxies, pool);
-    });
-
-    it('do not shuffle proxies when false', () => {
-        // make rotator with shuffle option set to false
-        let rotator = new ProxyRotator(test_proxies, { shuffle: false });
-        // get pool
-        let pool = rotator.getPool();
-        // check if the proxies are not shuffled
-        assert.deepEqual(test_proxies, pool);
-    });
-});
-

package/test/test.js

@@ -1,204 +0,0 @@
-import ProxyRotator from '../index.js'
-import assert from 'assert';
-import chai from 'chai';
-const expect = chai.expect
-
-// slogan: when you like a little risk with you proxies
-// proxies to test with 
-let test_proxies = [ '139.59.1.14:8080', '94.45.74.60:8080', 
-    '161.35.70.249:3128', '217.182.170.224:80', '222.138.76.6:9002',
-    '218.252.206.89:80',  '18.214.66.210:80', '120.234.203.171:9002' ]
-
-// check if it is able to read from difrent files
-describe('read form proxy files', () => {
-    // test if it can process a string point to a file
-    it('reading file with newlines', () => {
-        // make rotator with file with newlines
-        let proxies_file = './assets/http_proxies_with_newlines.txt';
-        // make rotator with file with newlines
-        let rotator_with_newlines = new ProxyRotator(proxies_file); 
-        // get pool
-        let pool = rotator_with_newlines.getPool();
-        assert.deepEqual(test_proxies, pool)
-    })
-    // test if it can process a file with spaces
-    it('reading file with spaces', () => {
-        // make rotator with file with spaces
-        let proxies_file = './assets/http_proxies_with_spaces.txt';
-        // make rotator with file with spaces
-        let rotator_with_spaces = new ProxyRotator(proxies_file);
-        // get pool
-        let pool = rotator_with_spaces.getPool();
-        assert.deepEqual(test_proxies, pool)
-    })
-    // test if it can process a file with commas
-    it('reading file with commas', () => {
-        // make rotator with file with commas
-        let proxies_file = './assets/http_proxies_with_commas.txt';
-        // make rotator with file with spaces
-        let rotator_with_commas = new ProxyRotator(proxies_file);
-        // get pool
-        let pool = rotator_with_commas.getPool();
-        assert.deepEqual(test_proxies, pool)
-    })
-});
-
-// check the functionality of the rotator
-describe('basic functionality', () => {
-    // test if it can process a string point to a file
-    it('add proxies array', () => {
-        // make rotator 
-        let rotator = new ProxyRotator(test_proxies);
-        // get pool
-        assert.deepEqual(test_proxies,rotator.getPool())
-    })
-    // adding one by one
-    it('add proxy one by one ', () => {
-        // make rotator
-        let rotator = new ProxyRotator();
-        // add proxies one by one
-        for(let proxy of test_proxies) rotator.add(proxy)
-        // test
-        assert.deepEqual(test_proxies,rotator.getPool())
-    })
-    // remove one by one
-    it('remove one', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies);
-        // remove proxies one by one
-        let removed = test_proxies.filter(e => e !== test_proxies[3])
-        rotator.remove(test_proxies[3])
-        // test
-        assert.deepEqual(removed, rotator.getPool())
-    })
-    // remove one by one
-    it('remove proxy one by one ', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies);
-        // remove proxies one by one
-        for(let proxy of test_proxies) rotator.remove(proxy)
-        // test
-        assert.deepEqual([], rotator.getPool())
-    })
-    // test if set dead works
-    it('set dead to graveyard', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies);
-        //  set dead
-        rotator.setDead(test_proxies[3])
-        // test
-        assert.deepEqual(
-            rotator.getGraveyard().includes(test_proxies[3]),
-            true
-        )
-    })
-    it('set dead remove from pool', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies);
-        //  set dead
-        rotator.setDead(test_proxies[3])
-        // test
-        assert.equal(
-            rotator.getPool().includes(test_proxies[3]),
-            false
-        )
-    })
-    it('resurect proxy from graveyard', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies);
-        //  set dead
-        rotator.setDead(test_proxies[3])
-        // set alive
-        rotator.resurect(test_proxies[3])
-        // test
-        assert.equal(rotator.getPool().includes(test_proxies[3]), true)
-    })
-    // test if set dead works
-    it('set dead proxy alive', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies);
-        //  set dead
-        rotator.setDead(test_proxies[3])
-        // set alive
-        rotator.setAlive(test_proxies[3])
-        // test
-        assert.equal(rotator.getPool().includes(test_proxies[3]), true)
-    })
-    // test if set dead works
-    it('get alive', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies);
-        // set alive
-        rotator.setAlive(test_proxies[3])
-        // get alive
-        let proxy = rotator.getAlive()
-        // test
-        assert.equal(proxy, test_proxies[3])
-    })
-    // test rotation
-    it('test rotation', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies);
-        let removed = test_proxies.filter(e => e !== test_proxies[3])
-        // set dead
-        rotator.setDead(test_proxies[3])
-        // test if every proxy returned with next matches removed 
-        for(let i = 0; i < removed.length; i++){
-            let proxy = rotator.next()
-            assert.equal(proxy, removed[i])
-        }
-    })
-    // test resurrect timer
-    it('resurect timer', function(done){
-        // set timeout
-        this.timeout(1100);
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies, { revive_timer: 1000 } );
-        let removed = test_proxies.filter(e => e !== test_proxies[3])
-        // set dead
-        rotator.setDead(test_proxies[3])
-        // wait for resurect timer
-        setTimeout(() => { 
-            assert.equal(
-                rotator.getPool().includes(test_proxies[3]),
-                true
-            );
-            done();
-        }, 1000);
-    })
-});
-
-// check return type of proxy
-describe('Test return type', () => {
-    // test if it return as obj when option is set
-    it('test default return string type', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies);
-        // remove proxies one by one
-        let proxy = rotator.next()
-        // test if it is string
-        assert.equal(typeof proxy, 'string')
-    })
-    it('test if it return object when passed returnAs obj', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies, { returnAs: 'object' });
-        // remove proxies one by one
-        let proxy = rotator.next()
-        // test if it is string
-        assert.equal(typeof proxy, 'object')
-    })
-    it('test if it return obj when passed as paramter', () => {
-        // make rotator
-        let rotator = new ProxyRotator(test_proxies);
-        // remove proxies one by one
-        let proxy = rotator.next({ returnAs:'obj'})
-        // test if it is string
-        assert.equal(typeof proxy, 'object')
-        // try to get sring
-        let proxy2 = rotator.next({ returnAs: 'str'})
-        // test if it is string
-        assert.equal(typeof proxy2, 'string')
-    })
-});
-
-

package/test/test_proxies.js

@@ -1,22 +0,0 @@
-import ProxyRotator from '../index.js'
-import assert from 'assert';
-import chai from 'chai';
-const expect = chai.expect
-
-// slogan: when you like a little risk with you proxies
-// proxies to test with 
-let test_proxies = [ '139.59.1.14:8080', '94.45.74.60:8080', 
-    '161.35.70.249:3128', '217.182.170.224:80', '222.138.76.6:9002',
-    '218.252.206.89:80',  '18.214.66.210:80', '120.234.203.171:9002' ]
-
-// check if it is able to read from difrent files
-describe('testing testing proxies', () => {
-    // test if it can process a string point to a file
-    it('testing proxeis', () => {
-        // make rotator 
-        let rotator = new ProxyRotator(test_proxies);
-        rotator.test()
-        assert.deepEqual(test_proxies,rotator.getPool())
-    })
-})
-