puppyproxy 1.1.0 → 1.1.2

package/README.md

@@ -26,14 +26,16 @@ const config = {
     },
     scraper: {
         use: true,
-        timeBetweenScrapes: 45 * 60e3,
+        timeBetweenScrapes: 45, // minutes
         maxProxiesToCheck: 5000, // the higher this is, the longer it takes, but the more proxies you get
         limitProxies: 500000,
-        timeoutPerProxy: 5,
+        timeoutPerProxy: 15, // seconds
         timeoutMax: 5,
         verbose: true, // for logging purposes
         logStatus: true, // shows a convenient summary every so often
         proxyTypes: ['socks4', 'socks5'], //['socks4', 'socks5', 'http', 'https']
+        method: 'ws', // 'ws' or 'http'
+        wsUrl: wsUrl,
 
         timeoutFetch: 10000, // ms
         timeoutWs: 7000,
@@ -48,6 +50,8 @@ log.highlight("TESTING: runCollector");
 
 await pp.init(true); // true forces run the collector
 
+console.log(pp.request.createProxyRequestPool().length, "proxies in the pool after init");
+
 log.highlight("TESTING: RequestClient");
 
 log.bold("----- pp.request.fetch")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "puppyproxy",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "description": "Node.js library for managing, scraping, and using proxies with fetch and WebSocket support",
   "type": "module",
   "types": "./src/index.d.ts",

package/src/PuppyProxy.js

@@ -97,6 +97,10 @@ export default class PuppyProxy {
         return this.request.createProxyRequest(url, options);
     }
 
+    createProxyRequestPool(url, options = {}) {
+        return this.request.createProxyRequestPool(url, options);
+    }
+
     /**
      * Trigger the proxy collection worker
      */

package/src/network/RequestClient.js

@@ -281,4 +281,28 @@ export default class RequestClient {
     createProxyRequest(url, options) {
         return new ProxyRequest(this, url, options);
     }
+
+    //return an array of ProxyRequest objects, one for each proxy
+    createProxyRequestPool(url, options = {}) {
+        const oldLOG = this.config.LOG;
+        this.config.LOG = {}; //suppress logs for this operation
+        let currentWithoutNew = 0;
+        const withoutNewLimit = 1000;
+        const proxyRequests = [];
+        const uniqueProxies = [];
+
+        while (currentWithoutNew < withoutNewLimit) {
+            const pr = this.createProxyRequest(url, options);
+            const proxy = pr.options.AGENT?.USINGSCRAPED;
+
+            if (proxy && !uniqueProxies.includes(proxy)) {
+                proxyRequests.push(pr);
+                uniqueProxies.push(proxy);
+                currentWithoutNew = 0;
+            } else currentWithoutNew++;
+        }
+
+        this.config.LOG = oldLOG;
+        return proxyRequests;
+    }
 }

package/src/providers/Scraper.js

@@ -96,7 +96,7 @@ export default class Scraper {
         let lastSaved = (Date.now() - getLastSavedTimestamp(this.outputPath)) / 1000;
         if (lastSaved && !force) {
             console.log("lastSaved", lastSaved);
-            let timeBetweenScrapes = this.config.scraper.timeBetweenScrapes * 60; // 25 mins
+            let timeBetweenScrapes = this.config.scraper.timeBetweenScrapes * 60e3; // 25 mins
             if (lastSaved < timeBetweenScrapes) {
                 console.log(`Proxies saved < ${timeBetweenScrapes}s ago, skipping.`);
                 return;