proxy-rotator-js 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +146 -0
- package/assets/http_proxies_with_commas.txt +1 -0
- package/assets/http_proxies_with_newlines.txt +8 -0
- package/assets/http_proxies_with_spaces.txt +1 -0
- package/index.js +2 -0
- package/package.json +104 -0
- package/src/Proxy.js +103 -0
- package/src/ProxyRotator.js +175 -0
- package/src/Queue.js +61 -0
- package/test/test.js +169 -0
package/README.md
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Proxy Rotator
|
|
2
|
+
=======
|
|
3
|
+
####
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
## Introduction
|
|
7
|
+
|
|
8
|
+
ProxyRotator is a JavaScript class that provides a mechanism for managing a pool of proxies and rotating them based on their availability and status.
|
|
9
|
+
|
|
10
|
+
## Installation
|
|
11
|
+
```
|
|
12
|
+
npm install proxy-rotator
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Properties
|
|
16
|
+
|
|
17
|
+
pool: Represents the pool of proxies as a queue.
|
|
18
|
+
|
|
19
|
+
graveyard: Stores proxies that are currently dead or inactive.
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
22
|
+
|
|
23
|
+
```javascript
|
|
24
|
+
|
|
25
|
+
import ProxyRoulette from 'proxy-rotator'
|
|
26
|
+
|
|
27
|
+
let proxies = ['proxy1', 'proxy2', 'proxy3']
|
|
28
|
+
|
|
29
|
+
let rotator = ProxyRotator(proxies, options={})
|
|
30
|
+
|
|
31
|
+
console.log( rotator.next() ) // 'proxy1'
|
|
32
|
+
console.log( rotator.next() ) // 'proxy2'
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Initializes a new instance of ProxyRotator with the given proxies and options. The proxies parameter can be a file path or an array of proxies. The options parameter allows customization of various settings such as revive timer, shuffling, protocol assumption, and more.
|
|
36
|
+
Methods
|
|
37
|
+
|
|
38
|
+
## Methods
|
|
39
|
+
|
|
40
|
+
```javascript
|
|
41
|
+
next() // Rotates the proxy by moving the front proxy to the end of the pool and returns it.
|
|
42
|
+
|
|
43
|
+
add(proxies) // Adds one or more proxies to the pool.
|
|
44
|
+
|
|
45
|
+
getAlive() // Retrieves a random alive proxy from the pool.
|
|
46
|
+
|
|
47
|
+
setAlive(proxy) // Sets a specific proxy to an alive state.
|
|
48
|
+
|
|
49
|
+
setDead(proxy) // Sets a specific proxy to a dead state and moves it to the graveyard.
|
|
50
|
+
|
|
51
|
+
resurrect(proxy) // Moves a proxy from the graveyard back to the pool.
|
|
52
|
+
|
|
53
|
+
getPool() // Returns an array of proxies in the pool.
|
|
54
|
+
|
|
55
|
+
getPoolSize() // Returns the number of proxies in the pool.
|
|
56
|
+
|
|
57
|
+
getGraveyard() // Returns an array of proxies in the graveyard (dead proxies).
|
|
58
|
+
|
|
59
|
+
getGraveyardSize() // Returns the number of proxies in the graveyard.
|
|
60
|
+
|
|
61
|
+
remove(proxy) // Removes one or more proxies from the pool.
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Getting Started
|
|
65
|
+
|
|
66
|
+
To use the ProxyRotator class in your JavaScript project, follow these steps:
|
|
67
|
+
|
|
68
|
+
Make sure you have Node.js and npm installed on your system.
|
|
69
|
+
|
|
70
|
+
```javascript
|
|
71
|
+
npm install proxy-rotator
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Import the ProxyRotator class into your JavaScript file using the following line of code:
|
|
75
|
+
|
|
76
|
+
```javascript
|
|
77
|
+
|
|
78
|
+
import ProxyRotator from './ProxyRotator.js';
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Create an instance of ProxyRotator by calling the constructor and providing the required parameters. For example:
|
|
82
|
+
|
|
83
|
+
```javascript
|
|
84
|
+
|
|
85
|
+
const proxies = ['proxy1:8000', 'proxy2:322', 'proxy3:543'];
|
|
86
|
+
const rotator = new ProxyRotator(proxies);
|
|
87
|
+
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Access the properties and methods of the ProxyRotator object using dot notation. For example:
|
|
91
|
+
|
|
92
|
+
```javascript
|
|
93
|
+
|
|
94
|
+
rotator.next();
|
|
95
|
+
// 'proxy1:500'
|
|
96
|
+
const poolSize = proxyRotator.getPoolSize();
|
|
97
|
+
// 3
|
|
98
|
+
const aliveProxy = proxyRotator.getAlive();
|
|
99
|
+
// 'proxy2:500'
|
|
100
|
+
proxyRotator.setAlive( 'proxy2:500' );
|
|
101
|
+
// null
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Examples
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
```javascript
|
|
108
|
+
|
|
109
|
+
import ProxyRotator from './ProxyRotator.js';
|
|
110
|
+
|
|
111
|
+
// Create an instance of ProxyRotator
|
|
112
|
+
const proxies = ['proxy1', 'proxy2', 'proxy3'];
|
|
113
|
+
const options = { revive_timer: 1800000, shuffle: true };
|
|
114
|
+
const proxyRotator = new ProxyRotator(proxies, options);
|
|
115
|
+
|
|
116
|
+
// Access the properties
|
|
117
|
+
console.log(proxyRotator.getGraveyard()); // Output: []
|
|
118
|
+
console.log(proxyRotator.getGraveyardSize()); // Output: 0
|
|
119
|
+
console.log(proxyRotator.getPool()); // Output: ['proxy1', 'proxy2', 'proxy3']
|
|
120
|
+
console.log(proxyRotator.getPoolSize()); // Output: 3
|
|
121
|
+
|
|
122
|
+
// Call the methods
|
|
123
|
+
proxyRotator.add('proxy4');
|
|
124
|
+
console.log(proxyRotator.getPool()); // Output: ['proxy1', 'proxy2', 'proxy3', 'proxy4']
|
|
125
|
+
proxyRotator.remove('proxy2');
|
|
126
|
+
console.log(proxyRotator.getPool()); // Output: ['proxy1', 'proxy3', 'proxy4']
|
|
127
|
+
const aliveProxy = proxyRotator.getAlive();
|
|
128
|
+
console.log(aliveProxy); // Output: 'proxy1'
|
|
129
|
+
proxyRotator.setAlive('proxy3');
|
|
130
|
+
proxyRotator.next(); // prox1
|
|
131
|
+
proxyRotator.next(); // prox3
|
|
132
|
+
proxyRotator.next(); // prox4
|
|
133
|
+
console.log(proxyRotator.getPool()); // Output: ['proxy1', 'proxy4', 'proxy3']
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### Contributing
|
|
137
|
+
|
|
138
|
+
If you would like to contribute to the ProxyRotator project, you can fork the repository and make your desired changes. Feel free to submit a pull request with your improvements or bug fixes. We appreciate your contributions!
|
|
139
|
+
License
|
|
140
|
+
|
|
141
|
+
The ProxyRotator class is released under the MIT License. You can freely use and modify it in your projects. Please refer to the license file for more information.
|
|
142
|
+
Contact
|
|
143
|
+
|
|
144
|
+
If you have any questions, suggestions, or feedback regarding the ProxyRotator class, please me =) Goran Topic @ telegonicaxx@live.com. We appreciate your input and are happy to assist you.
|
|
145
|
+
|
|
146
|
+
Thank you for using the ProxyRotator class. We hope it helps simplify your proxy management and rotation tasks.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
139.59.1.14:8080, 94.45.74.60:8080, 161.35.70.249:3128, 217.182.170.224:80, 222.138.76.6:9002, 218.252.206.89:80, 18.214.66.210:80, 120.234.203.171:9002
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
139.59.1.14:8080 94.45.74.60:8080 161.35.70.249:3128 217.182.170.224:80 222.138.76.6:9002 218.252.206.89:80 18.214.66.210:80 120.234.203.171:9002
|
package/index.js
ADDED
package/package.json
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "proxy-rotator-js",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Proxy Rotator",
|
|
6
|
+
"main": "index.js",
|
|
7
|
+
"directories": {
|
|
8
|
+
"test": "test"
|
|
9
|
+
},
|
|
10
|
+
"dependencies": {
|
|
11
|
+
"ansi-colors": "^4.1.1",
|
|
12
|
+
"ansi-regex": "^5.0.1",
|
|
13
|
+
"ansi-styles": "^4.3.0",
|
|
14
|
+
"anymatch": "^3.1.3",
|
|
15
|
+
"argparse": "^2.0.1",
|
|
16
|
+
"assertion-error": "^1.1.0",
|
|
17
|
+
"balanced-match": "^1.0.2",
|
|
18
|
+
"binary-extensions": "^2.2.0",
|
|
19
|
+
"brace-expansion": "^2.0.1",
|
|
20
|
+
"braces": "^3.0.2",
|
|
21
|
+
"browser-stdout": "^1.3.1",
|
|
22
|
+
"camelcase": "^6.3.0",
|
|
23
|
+
"chai": "^4.3.7",
|
|
24
|
+
"chalk": "^4.1.2",
|
|
25
|
+
"check-error": "^1.0.2",
|
|
26
|
+
"chokidar": "^3.5.3",
|
|
27
|
+
"cliui": "^7.0.4",
|
|
28
|
+
"color-convert": "^2.0.1",
|
|
29
|
+
"color-name": "^1.1.4",
|
|
30
|
+
"concat-map": "^0.0.1",
|
|
31
|
+
"debug": "^4.3.4",
|
|
32
|
+
"decamelize": "^4.0.0",
|
|
33
|
+
"deep-eql": "^4.1.3",
|
|
34
|
+
"diff": "^5.0.0",
|
|
35
|
+
"emoji-regex": "^8.0.0",
|
|
36
|
+
"escalade": "^3.1.1",
|
|
37
|
+
"escape-string-regexp": "^4.0.0",
|
|
38
|
+
"files-js": "^1.0.1",
|
|
39
|
+
"fill-range": "^7.0.1",
|
|
40
|
+
"find-up": "^5.0.0",
|
|
41
|
+
"flat": "^5.0.2",
|
|
42
|
+
"fs.realpath": "^1.0.0",
|
|
43
|
+
"get-caller-file": "^2.0.5",
|
|
44
|
+
"get-func-name": "^2.0.0",
|
|
45
|
+
"glob": "^7.2.0",
|
|
46
|
+
"glob-parent": "^5.1.2",
|
|
47
|
+
"has-flag": "^4.0.0",
|
|
48
|
+
"he": "^1.2.0",
|
|
49
|
+
"inflight": "^1.0.6",
|
|
50
|
+
"inherits": "^2.0.4",
|
|
51
|
+
"is-binary-path": "^2.1.0",
|
|
52
|
+
"is-extglob": "^2.1.1",
|
|
53
|
+
"is-fullwidth-code-point": "^3.0.0",
|
|
54
|
+
"is-glob": "^4.0.3",
|
|
55
|
+
"is-number": "^7.0.0",
|
|
56
|
+
"is-plain-obj": "^2.1.0",
|
|
57
|
+
"is-unicode-supported": "^0.1.0",
|
|
58
|
+
"js-yaml": "^4.1.0",
|
|
59
|
+
"locate-path": "^6.0.0",
|
|
60
|
+
"log-symbols": "^4.1.0",
|
|
61
|
+
"loupe": "^2.3.6",
|
|
62
|
+
"minimatch": "^5.0.1",
|
|
63
|
+
"ms": "^2.1.3",
|
|
64
|
+
"nanoid": "^3.3.3",
|
|
65
|
+
"normalize-path": "^3.0.0",
|
|
66
|
+
"object-hash": "^3.0.0",
|
|
67
|
+
"once": "^1.4.0",
|
|
68
|
+
"p-limit": "^3.1.0",
|
|
69
|
+
"p-locate": "^5.0.0",
|
|
70
|
+
"path-exists": "^4.0.0",
|
|
71
|
+
"path-is-absolute": "^1.0.1",
|
|
72
|
+
"pathval": "^1.1.1",
|
|
73
|
+
"picomatch": "^2.3.1",
|
|
74
|
+
"randombytes": "^2.1.0",
|
|
75
|
+
"readdirp": "^3.6.0",
|
|
76
|
+
"require-directory": "^2.1.1",
|
|
77
|
+
"rimraf": "^3.0.2",
|
|
78
|
+
"safe-buffer": "^5.2.1",
|
|
79
|
+
"serialize-javascript": "^6.0.0",
|
|
80
|
+
"string-width": "^4.2.3",
|
|
81
|
+
"strip-ansi": "^6.0.1",
|
|
82
|
+
"strip-json-comments": "^3.1.1",
|
|
83
|
+
"supports-color": "^8.1.1",
|
|
84
|
+
"tmp": "^0.2.1",
|
|
85
|
+
"to-regex-range": "^5.0.1",
|
|
86
|
+
"type-detect": "^4.0.8",
|
|
87
|
+
"workerpool": "^6.2.1",
|
|
88
|
+
"wrap-ansi": "^7.0.0",
|
|
89
|
+
"wrappy": "^1.0.2",
|
|
90
|
+
"y18n": "^5.0.8",
|
|
91
|
+
"yargs": "^16.2.0",
|
|
92
|
+
"yargs-parser": "^20.2.4",
|
|
93
|
+
"yargs-unparser": "^2.0.0",
|
|
94
|
+
"yocto-queue": "^0.1.0"
|
|
95
|
+
},
|
|
96
|
+
"devDependencies": {
|
|
97
|
+
"mocha": "^10.2.0"
|
|
98
|
+
},
|
|
99
|
+
"scripts": {
|
|
100
|
+
"test": "mocha"
|
|
101
|
+
},
|
|
102
|
+
"author": "Goran Topic",
|
|
103
|
+
"license": "ISC"
|
|
104
|
+
}
|
package/src/Proxy.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
class Proxy {
|
|
2
|
+
constructor(proxy, protocol = null) {
|
|
3
|
+
// if proxy string starts with protocol
|
|
4
|
+
if( proxy.includes('://') ){
|
|
5
|
+
this.protocol = proxy.split('://')[0];
|
|
6
|
+
this.ip = proxy.split('://')[1].split(':')[0];
|
|
7
|
+
this.port = proxy.split('://')[1].split(':')[1];
|
|
8
|
+
} else { // if proxy string does not start with protocol
|
|
9
|
+
this.protocol = protocol;
|
|
10
|
+
this.ip = proxy.split(':')[0];
|
|
11
|
+
this.port = proxy.split(':')[1];
|
|
12
|
+
}
|
|
13
|
+
// the proxy
|
|
14
|
+
this.proxy = `${(this.protocol)? this.protocol+'://' : ''}${this.ip}:${this.port}`;
|
|
15
|
+
// status can be 'new', 'alive', 'dead'
|
|
16
|
+
this.status = 'new';
|
|
17
|
+
this.changeTimeStamp = Date.now();
|
|
18
|
+
}
|
|
19
|
+
// method to return as string
|
|
20
|
+
proxy() {
|
|
21
|
+
return `${(this.protocol)? this.protocol+'://' : ''}${this.ip}:${this.port}`;
|
|
22
|
+
}
|
|
23
|
+
toString() {
|
|
24
|
+
return `${(this.protocol)? this.protocol+'://' : ''}${this.ip}:${this.port}`;
|
|
25
|
+
}
|
|
26
|
+
// method to return as obj
|
|
27
|
+
get() {
|
|
28
|
+
return {
|
|
29
|
+
protocol: this.protocol,
|
|
30
|
+
ip: this.ip,
|
|
31
|
+
port: this.port,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
// method to return as obj
|
|
35
|
+
obj() {
|
|
36
|
+
return {
|
|
37
|
+
protocol: this.protocol,
|
|
38
|
+
ip: this.ip,
|
|
39
|
+
port: this.port,
|
|
40
|
+
status: this.status,
|
|
41
|
+
changeTimeStamp: this.changeTimeStamp
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
// mark proxy as dead
|
|
45
|
+
kill() {
|
|
46
|
+
this.status = 'dead';
|
|
47
|
+
this.changeTimeStamp = Date.now();
|
|
48
|
+
}
|
|
49
|
+
setDead() {
|
|
50
|
+
this.status = 'dead'
|
|
51
|
+
this.changeTimeStamp = Date.now();
|
|
52
|
+
}
|
|
53
|
+
setAlive(){
|
|
54
|
+
this.status = 'alive'
|
|
55
|
+
this.changeTimeStamp = Date.now();
|
|
56
|
+
}
|
|
57
|
+
setNew() {
|
|
58
|
+
this.status = 'new'
|
|
59
|
+
this.changeTimeStamp = Date.now();
|
|
60
|
+
}
|
|
61
|
+
isDead() {
|
|
62
|
+
return this.status === 'dead';
|
|
63
|
+
}
|
|
64
|
+
isAlive() {
|
|
65
|
+
return this.status === 'alive';
|
|
66
|
+
}
|
|
67
|
+
isNew() {
|
|
68
|
+
return this.status === 'new';
|
|
69
|
+
}
|
|
70
|
+
revive() {
|
|
71
|
+
this.status = 'alive';
|
|
72
|
+
this.changeTimeStamp = Date.now();
|
|
73
|
+
}
|
|
74
|
+
reset() {
|
|
75
|
+
this.status = 'new';
|
|
76
|
+
this.changeTimeStamp = Date.now();
|
|
77
|
+
}
|
|
78
|
+
status() {
|
|
79
|
+
return this.status;
|
|
80
|
+
}
|
|
81
|
+
equals(proxy) {
|
|
82
|
+
if (typeof proxy === 'string')
|
|
83
|
+
proxy = new Proxy(proxy);
|
|
84
|
+
return (this.ip === proxy.ip && this.port === proxy.port);
|
|
85
|
+
}
|
|
86
|
+
timeSinceStatusChange() {
|
|
87
|
+
return Date.now() - this.changeTimeStamp;
|
|
88
|
+
}
|
|
89
|
+
print() {
|
|
90
|
+
console.log(this.get());
|
|
91
|
+
}
|
|
92
|
+
getIp() {
|
|
93
|
+
return this.ip;
|
|
94
|
+
}
|
|
95
|
+
getPort() {
|
|
96
|
+
return this.port;
|
|
97
|
+
}
|
|
98
|
+
getProtocol() {
|
|
99
|
+
return this.protocol;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export default Proxy;
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import Queue from './Queue.js';
|
|
3
|
+
import Proxy from './Proxy.js';
|
|
4
|
+
|
|
5
|
+
class ProxyRotator {
|
|
6
|
+
constructor(proxies, options={} ){
|
|
7
|
+
this.pool = new Queue();
|
|
8
|
+
this.graveyard = [];
|
|
9
|
+
// examine proxies passed
|
|
10
|
+
// check if it is a file path
|
|
11
|
+
if( typeof proxies === 'string' ){
|
|
12
|
+
// parse file
|
|
13
|
+
proxies = this._parseFile(proxies);
|
|
14
|
+
// add proxies to queue
|
|
15
|
+
proxies.forEach( p => this._add(p) );
|
|
16
|
+
}else if( this._isArray(proxies) ){
|
|
17
|
+
// add proxies to queue
|
|
18
|
+
proxies.forEach( p => this._add(p) );
|
|
19
|
+
}
|
|
20
|
+
// handle options
|
|
21
|
+
let { revive_timer, shuffle, protocol, assume_aliveness, check_on_next } = options;
|
|
22
|
+
// how long to wait before reviving a dead proxy
|
|
23
|
+
// default: 30 minutes
|
|
24
|
+
this.revive_timer = revive_timer ?? 1000 * 60 * 30;
|
|
25
|
+
// assume a a protocol for all proxies
|
|
26
|
+
this.protocol = protocol ?? null;
|
|
27
|
+
// shuffle the proxies before adding them to the queue
|
|
28
|
+
this.shuffle = shuffle ?? true;
|
|
29
|
+
// assume all proxies are alive when first added instead of 'new'
|
|
30
|
+
this.assume_aliveness = assume_aliveness ?? false;
|
|
31
|
+
// check if proxies are alive when they are added to the queue
|
|
32
|
+
this.check_on_next = check_on_next ?? false;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
getGraveyard(){ return this.graveyard.map(p => p.proxy) }
|
|
36
|
+
|
|
37
|
+
getGraveyardSize(){ return this.graveyard.length }
|
|
38
|
+
|
|
39
|
+
getPool(){ return this.pool.toArray().map(p => p.proxy) }
|
|
40
|
+
|
|
41
|
+
getPoolSize(){ return this.pool.size }
|
|
42
|
+
|
|
43
|
+
add(proxies){ // add proxy to queue
|
|
44
|
+
if(this._isArray(proxies)) // if passed an array
|
|
45
|
+
for(let proxy of proxies) this._add(proxy);
|
|
46
|
+
else // single file
|
|
47
|
+
this._add(proxies);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
_add(proxy){ // add proxy to queue
|
|
51
|
+
let p = new Proxy(proxy);
|
|
52
|
+
this.pool.enqueue(p);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
remove(proxy){ // remove proxy from queue
|
|
56
|
+
if(this._isArray(proxy)) // if passed an array
|
|
57
|
+
for(let p of proxy) this._remove(p);
|
|
58
|
+
else // single file
|
|
59
|
+
this._remove(proxy);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
_remove(proxy){ // remove proxy from queue
|
|
63
|
+
this.pool.toArray().forEach( (p,i) => {
|
|
64
|
+
if(p.equals(proxy)) this.pool.remove(i);
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
getAlive(){ // get a random alive proxy
|
|
69
|
+
let proxies = this.pool.toArray();
|
|
70
|
+
for( let proxy of proxies )
|
|
71
|
+
if(proxy.isAlive()) return proxy.proxy;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
setAlive(proxy){ // set a proxy to alive
|
|
75
|
+
let proxies = this.pool.toArray()
|
|
76
|
+
for( let p of proxies ) // if proxy is in queue
|
|
77
|
+
if(p.equals(proxy)) return p.setAlive();
|
|
78
|
+
// if proxy is in graveyard
|
|
79
|
+
for( let p of this.graveyard )
|
|
80
|
+
if(p.equals(proxy)) return this.resurect(p);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
resurect( proxy ){
|
|
84
|
+
// get proxy from graveyard
|
|
85
|
+
let p = this.graveyard.find( p => p.equals(proxy) );
|
|
86
|
+
// if proxy is not in graveyard
|
|
87
|
+
if(!p) return;
|
|
88
|
+
// remove from graveyard
|
|
89
|
+
this.graveyard = this.graveyard.filter( p => !p.equals(proxy) );
|
|
90
|
+
// set as new
|
|
91
|
+
p.setNew();
|
|
92
|
+
// add to queue
|
|
93
|
+
this.pool.enqueue(p);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
setDead(proxy){ // set a proxy to dead
|
|
97
|
+
this.pool.toArray().forEach( (p,i) => {
|
|
98
|
+
if(p.equals(proxy)){
|
|
99
|
+
p.setDead(); // set to dead
|
|
100
|
+
// remove from queue
|
|
101
|
+
this.pool.remove(i);
|
|
102
|
+
// add to graveyard
|
|
103
|
+
this.graveyard.push(p);
|
|
104
|
+
}
|
|
105
|
+
})
|
|
106
|
+
// revive proxy after revive_timer
|
|
107
|
+
setTimeout( () => this.resurect(proxy), this.revive_timer );
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
next(){
|
|
111
|
+
// resurect a proxy from the graveyard
|
|
112
|
+
if(this.check_on_next) _resurection();
|
|
113
|
+
// if there are no proxies in the pool
|
|
114
|
+
if(this.pool.size === 0) return null ;
|
|
115
|
+
// remove from front
|
|
116
|
+
let proxy = this.pool.dequeue();
|
|
117
|
+
// add to back
|
|
118
|
+
this.pool.enqueue(proxy);
|
|
119
|
+
// return
|
|
120
|
+
return proxy
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/* Randomize array in-place using Durstenfeld shuffle algorithm */
|
|
124
|
+
_shuffleArray(array) {
|
|
125
|
+
for (var i = array.length - 1; i > 0; i--) {
|
|
126
|
+
var j = Math.floor(Math.random() * (i + 1));
|
|
127
|
+
var temp = array[i];
|
|
128
|
+
array[i] = array[j];
|
|
129
|
+
array[j] = temp;
|
|
130
|
+
}
|
|
131
|
+
return array;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
_parseFile(filename) {
|
|
135
|
+
// read file
|
|
136
|
+
let str = fs.readFileSync(filename, 'utf8');
|
|
137
|
+
// this function is able to handle multiples types of files
|
|
138
|
+
let strList = str.split('\n')
|
|
139
|
+
// remove lines that are empty
|
|
140
|
+
strList = strList.filter(s=>s.length>0);
|
|
141
|
+
// if strList is has only one element
|
|
142
|
+
if(strList.length === 1)
|
|
143
|
+
strList = str.split(' ')
|
|
144
|
+
// if strList is has only one check to separate by comma
|
|
145
|
+
if(strList.length === 1)
|
|
146
|
+
strList = str.split(',')
|
|
147
|
+
// remove all the commas from the strings
|
|
148
|
+
strList = strList.map(s=>s.replace(',',''))
|
|
149
|
+
// parse list of strings into
|
|
150
|
+
strList = strList
|
|
151
|
+
.map(s=>s.trim())
|
|
152
|
+
.filter(s=>s.length>0);
|
|
153
|
+
// return proxies
|
|
154
|
+
return strList;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
_isArray(arrayValue){
|
|
159
|
+
return ( arrayValue &&
|
|
160
|
+
(typeof arrayValue === 'object') &&
|
|
161
|
+
(arrayValue.constructor === Array) );
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
_resurection(){
|
|
165
|
+
// look for proxies that have been
|
|
166
|
+
// dead for a long than the resurection time
|
|
167
|
+
// and revive them
|
|
168
|
+
for( let proxy of this.graveyard )
|
|
169
|
+
if( proxy.isDead() && proxy.timeSinceStatusChange >= this.revive_timer )
|
|
170
|
+
this.resurect(proxy);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export default ProxyRotator
|
package/src/Queue.js
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
class Queue {
|
|
2
|
+
constructor() {
|
|
3
|
+
this.items = {}
|
|
4
|
+
this.frontIndex = 0
|
|
5
|
+
this.backIndex = 0
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
enqueue(item) {
|
|
9
|
+
this.items[this.backIndex] = item
|
|
10
|
+
this.backIndex++
|
|
11
|
+
return item + ' inserted'
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
dequeue() {
|
|
15
|
+
const item = this.items[this.frontIndex]
|
|
16
|
+
delete this.items[this.frontIndex]
|
|
17
|
+
this.frontIndex++
|
|
18
|
+
return item
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// remove value while maintaining order
|
|
22
|
+
remove( index ) {
|
|
23
|
+
if (index < 0 || index >= this.backIndex) {
|
|
24
|
+
return 'Invalid index'
|
|
25
|
+
}
|
|
26
|
+
const item = this.items[index]
|
|
27
|
+
for (let i = index; i < this.backIndex; i++) {
|
|
28
|
+
this.items[i] = this.items[i + 1]
|
|
29
|
+
}
|
|
30
|
+
delete this.items[this.backIndex - 1]
|
|
31
|
+
this.backIndex--
|
|
32
|
+
return item
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
peek() {
|
|
36
|
+
return this.items[this.frontIndex]
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
printQueue() {
|
|
40
|
+
return this.items;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// return array of items in the order they were added
|
|
44
|
+
toArray() {
|
|
45
|
+
let arr = [];
|
|
46
|
+
for(let i = this.frontIndex; i < this.backIndex; i++)
|
|
47
|
+
arr.push(this.items[i]);
|
|
48
|
+
return arr;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// return object of items in the order they were added
|
|
52
|
+
toObject() {
|
|
53
|
+
let obj = {};
|
|
54
|
+
for(let i = this.frontIndex; i < this.backIndex; i++)
|
|
55
|
+
obj[i] = this.items[i];
|
|
56
|
+
return obj;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export default Queue;
|
package/test/test.js
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import ProxyRotator from '../index.js'
|
|
2
|
+
import assert from 'assert';
|
|
3
|
+
import chai from 'chai';
|
|
4
|
+
const expect = chai.expect
|
|
5
|
+
|
|
6
|
+
// slogan: when you like a little risk with you proxies
|
|
7
|
+
// proxies to test with
|
|
8
|
+
let test_proxies = [ '139.59.1.14:8080', '94.45.74.60:8080',
|
|
9
|
+
'161.35.70.249:3128', '217.182.170.224:80', '222.138.76.6:9002',
|
|
10
|
+
'218.252.206.89:80', '18.214.66.210:80', '120.234.203.171:9002' ]
|
|
11
|
+
|
|
12
|
+
// check if it is able to read from difrent files
|
|
13
|
+
describe('read form proxy files', () => {
|
|
14
|
+
// test if it can process a string point to a file
|
|
15
|
+
it('reading file with newlines', () => {
|
|
16
|
+
// make rotator with file with newlines
|
|
17
|
+
let proxies_file = './assets/http_proxies_with_newlines.txt';
|
|
18
|
+
// make rotator with file with newlines
|
|
19
|
+
let rotator_with_newlines = new ProxyRotator(proxies_file);
|
|
20
|
+
// get pool
|
|
21
|
+
let pool = rotator_with_newlines.getPool();
|
|
22
|
+
assert.deepEqual(test_proxies, pool)
|
|
23
|
+
})
|
|
24
|
+
// test if it can process a file with spaces
|
|
25
|
+
it('reading file with spaces', () => {
|
|
26
|
+
// make rotator with file with spaces
|
|
27
|
+
let proxies_file = './assets/http_proxies_with_spaces.txt';
|
|
28
|
+
// make rotator with file with spaces
|
|
29
|
+
let rotator_with_spaces = new ProxyRotator(proxies_file);
|
|
30
|
+
// get pool
|
|
31
|
+
let pool = rotator_with_spaces.getPool();
|
|
32
|
+
assert.deepEqual(test_proxies, pool)
|
|
33
|
+
})
|
|
34
|
+
// test if it can process a file with commas
|
|
35
|
+
it('reading file with commas', () => {
|
|
36
|
+
// make rotator with file with commas
|
|
37
|
+
let proxies_file = './assets/http_proxies_with_commas.txt';
|
|
38
|
+
// make rotator with file with spaces
|
|
39
|
+
let rotator_with_commas = new ProxyRotator(proxies_file);
|
|
40
|
+
// get pool
|
|
41
|
+
let pool = rotator_with_commas.getPool();
|
|
42
|
+
assert.deepEqual(test_proxies, pool)
|
|
43
|
+
})
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
// check the functionality of the rotator
|
|
47
|
+
describe('basic functionality', () => {
|
|
48
|
+
// test if it can process a string point to a file
|
|
49
|
+
it('add proxies array', () => {
|
|
50
|
+
// make rotator with file with newlines
|
|
51
|
+
let rotator = new ProxyRotator(test_proxies);
|
|
52
|
+
// get pool
|
|
53
|
+
assert.deepEqual(test_proxies,rotator.getPool())
|
|
54
|
+
})
|
|
55
|
+
// adding one by one
|
|
56
|
+
it('add proxy one by one ', () => {
|
|
57
|
+
// make rotator with file with newlines
|
|
58
|
+
let rotator = new ProxyRotator();
|
|
59
|
+
// add proxies one by one
|
|
60
|
+
for(let proxy of test_proxies) rotator.add(proxy)
|
|
61
|
+
// test
|
|
62
|
+
assert.deepEqual(test_proxies,rotator.getPool())
|
|
63
|
+
})
|
|
64
|
+
// remove one by one
|
|
65
|
+
it('remove one', () => {
|
|
66
|
+
// make rotator
|
|
67
|
+
let rotator = new ProxyRotator(test_proxies);
|
|
68
|
+
// remove proxies one by one
|
|
69
|
+
let removed = test_proxies.filter(e => e !== test_proxies[3])
|
|
70
|
+
rotator.remove(test_proxies[3])
|
|
71
|
+
// test
|
|
72
|
+
assert.deepEqual(removed, rotator.getPool())
|
|
73
|
+
})
|
|
74
|
+
// remove one by one
|
|
75
|
+
it('remove proxy one by one ', () => {
|
|
76
|
+
// make rotator
|
|
77
|
+
let rotator = new ProxyRotator(test_proxies);
|
|
78
|
+
// remove proxies one by one
|
|
79
|
+
for(let proxy of test_proxies) rotator.remove(proxy)
|
|
80
|
+
// test
|
|
81
|
+
assert.deepEqual([], rotator.getPool())
|
|
82
|
+
})
|
|
83
|
+
// test if set dead works
|
|
84
|
+
it('set dead to graveyard', () => {
|
|
85
|
+
// make rotator
|
|
86
|
+
let rotator = new ProxyRotator(test_proxies);
|
|
87
|
+
// set dead
|
|
88
|
+
rotator.setDead(test_proxies[3])
|
|
89
|
+
// test
|
|
90
|
+
assert.deepEqual(
|
|
91
|
+
rotator.getGraveyard().includes(test_proxies[3]),
|
|
92
|
+
true
|
|
93
|
+
)
|
|
94
|
+
})
|
|
95
|
+
it('set dead remove from pool', () => {
|
|
96
|
+
// make rotator
|
|
97
|
+
let rotator = new ProxyRotator(test_proxies);
|
|
98
|
+
// set dead
|
|
99
|
+
rotator.setDead(test_proxies[3])
|
|
100
|
+
// test
|
|
101
|
+
assert.equal(
|
|
102
|
+
rotator.getPool().includes(test_proxies[3]),
|
|
103
|
+
false
|
|
104
|
+
)
|
|
105
|
+
})
|
|
106
|
+
it('resurect proxy from graveyard', () => {
|
|
107
|
+
// make rotator
|
|
108
|
+
let rotator = new ProxyRotator(test_proxies);
|
|
109
|
+
// set dead
|
|
110
|
+
rotator.setDead(test_proxies[3])
|
|
111
|
+
// set alive
|
|
112
|
+
rotator.resurect(test_proxies[3])
|
|
113
|
+
// test
|
|
114
|
+
assert.equal(rotator.getPool().includes(test_proxies[3]), true)
|
|
115
|
+
})
|
|
116
|
+
// test if set dead works
|
|
117
|
+
it('set dead proxy alive', () => {
|
|
118
|
+
// make rotator
|
|
119
|
+
let rotator = new ProxyRotator(test_proxies);
|
|
120
|
+
// set dead
|
|
121
|
+
rotator.setDead(test_proxies[3])
|
|
122
|
+
// set alive
|
|
123
|
+
rotator.setAlive(test_proxies[3])
|
|
124
|
+
// test
|
|
125
|
+
assert.equal(rotator.getPool().includes(test_proxies[3]), true)
|
|
126
|
+
})
|
|
127
|
+
// test if set dead works
|
|
128
|
+
it('get alive', () => {
|
|
129
|
+
// make rotator
|
|
130
|
+
let rotator = new ProxyRotator(test_proxies);
|
|
131
|
+
// set alive
|
|
132
|
+
rotator.setAlive(test_proxies[3])
|
|
133
|
+
// get alive
|
|
134
|
+
let proxy = rotator.getAlive()
|
|
135
|
+
// test
|
|
136
|
+
assert.equal(proxy, test_proxies[3])
|
|
137
|
+
})
|
|
138
|
+
// test rotation
|
|
139
|
+
it('test rotation', () => {
|
|
140
|
+
// make rotator
|
|
141
|
+
let rotator = new ProxyRotator(test_proxies);
|
|
142
|
+
let removed = test_proxies.filter(e => e !== test_proxies[3])
|
|
143
|
+
// set dead
|
|
144
|
+
rotator.setDead(test_proxies[3])
|
|
145
|
+
// test if every proxy returned with next matches removed
|
|
146
|
+
for(let i = 0; i < removed.length; i++){
|
|
147
|
+
let proxy = rotator.next()
|
|
148
|
+
assert.equal(proxy, removed[i])
|
|
149
|
+
}
|
|
150
|
+
})
|
|
151
|
+
// test resurrect timer
|
|
152
|
+
it('resurect timer', function(done){
|
|
153
|
+
// set timeout
|
|
154
|
+
this.timeout(1100);
|
|
155
|
+
// make rotator
|
|
156
|
+
let rotator = new ProxyRotator(test_proxies, { revive_timer: 1000 } );
|
|
157
|
+
let removed = test_proxies.filter(e => e !== test_proxies[3])
|
|
158
|
+
// set dead
|
|
159
|
+
rotator.setDead(test_proxies[3])
|
|
160
|
+
// wait for resurect timer
|
|
161
|
+
setTimeout(() => {
|
|
162
|
+
assert.equal(
|
|
163
|
+
rotator.getPool().includes(test_proxies[3]),
|
|
164
|
+
true
|
|
165
|
+
);
|
|
166
|
+
done();
|
|
167
|
+
}, 1000);
|
|
168
|
+
})
|
|
169
|
+
});
|