puppyproxy 1.1.4 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/PuppyProxy.js +7 -0
- package/src/providers/Scraper.js +2 -3
package/package.json
CHANGED
package/src/PuppyProxy.js
CHANGED
|
@@ -49,6 +49,13 @@ export default class PuppyProxy {
|
|
|
49
49
|
// Load scraped proxies into memory
|
|
50
50
|
await this.scraper.runCollector(forceCollector);
|
|
51
51
|
await this.scraper.loadScrapedProxies();
|
|
52
|
+
|
|
53
|
+
if (this.config.scraper?.autoCollectProxies) {
|
|
54
|
+
log.info(id, `Auto proxy collection enabled. Running every ${this.config.scraper.autoCollectProxies} minutes.`);
|
|
55
|
+
setInterval(async () => {
|
|
56
|
+
await this.scraper.runCollector();
|
|
57
|
+
}, (this.config.scraper.autoCollectProxies) * 60e3); // minutes to ms
|
|
58
|
+
}
|
|
52
59
|
}
|
|
53
60
|
|
|
54
61
|
/**
|
package/src/providers/Scraper.js
CHANGED
|
@@ -91,12 +91,11 @@ export default class Scraper {
|
|
|
91
91
|
return null;
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
-
// Worker Orchestration
|
|
95
94
|
async runCollector(force = false) {
|
|
96
|
-
let lastSaved = (Date.now() - getLastSavedTimestamp(this.outputPath)) / 1000;
|
|
95
|
+
let lastSaved = (Date.now() - getLastSavedTimestamp(this.outputPath)) / 1000; // ms to seconds
|
|
97
96
|
if (lastSaved && !force) {
|
|
98
97
|
console.log("lastSaved", lastSaved);
|
|
99
|
-
let timeBetweenScrapes = this.config.scraper.timeBetweenScrapes *
|
|
98
|
+
let timeBetweenScrapes = this.config.scraper.timeBetweenScrapes * 60; // mins to seconds
|
|
100
99
|
if (lastSaved < timeBetweenScrapes) {
|
|
101
100
|
console.log(`Proxies saved < ${timeBetweenScrapes}s ago, skipping.`);
|
|
102
101
|
return;
|