puppyproxy 1.1.1 → 1.1.3

package/README.md

@@ -50,6 +50,8 @@ log.highlight("TESTING: runCollector");
 
 await pp.init(true); // true forces run the collector
 
+console.log(pp.request.createProxyRequestPool().length, "proxies in the pool after init");
+
 log.highlight("TESTING: RequestClient");
 
 log.bold("----- pp.request.fetch")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "puppyproxy",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "description": "Node.js library for managing, scraping, and using proxies with fetch and WebSocket support",
   "type": "module",
   "types": "./src/index.d.ts",

package/src/PuppyProxy.js

@@ -97,6 +97,10 @@ export default class PuppyProxy {
         return this.request.createProxyRequest(url, options);
     }
 
+    createProxyRequestPool(url, options = {}) {
+        return this.request.createProxyRequestPool(url, options);
+    }
+
     /**
      * Trigger the proxy collection worker
      */

package/src/network/RequestClient.js

@@ -1,6 +1,8 @@
 import log from 'puppylog';
 import id from 'puppyid';
 import { originals } from '../constants.js';
+import scrambled from 'puppyscrambled';
+import { createBasicAgent } from '../utils.js';
 
 export class ProxyRequest {
     constructor(requestClient, url, options = {}) {
@@ -281,4 +283,31 @@ export default class RequestClient {
     createProxyRequest(url, options) {
         return new ProxyRequest(this, url, options);
     }
+
+    //return an array of ProxyRequest objects, one for each proxy
+    createProxyRequestPool(url, options = {}) {
+        const oldLOG = this.config.LOG;
+        this.config.LOG = {}; //suppress logs for this operation
+        let currentWithoutNew = 0;
+        const withoutNewLimit = 1000;
+        const proxyRequests = [];
+        const uniqueProxies = [];
+
+        while (currentWithoutNew < withoutNewLimit) {
+            const pr = this.createProxyRequest(url, { ...options });
+            const proxy = pr.options.AGENT?.USINGSCRAPED;
+
+            if (proxy && !uniqueProxies.includes(proxy)) {
+                pr.id = scrambled.getScrambled();
+                console.log(pr.id, pr.options.AGENT.___id, "added proxy to pool:", proxy);
+                pr.options.AGENT.___id = pr.id;
+                proxyRequests.push(pr);
+                uniqueProxies.push(proxy);
+                currentWithoutNew = 0;
+            } else currentWithoutNew++;
+        }
+
+        this.config.LOG = oldLOG;
+        return proxyRequests;
+    }
 }

package/src/utils.js

@@ -4,9 +4,14 @@ import { HttpsProxyAgent } from 'https-proxy-agent';
 
 export function createBasicAgent(proxyUrl) {
     if (!proxyUrl) return null;
+
+    let proxy = null;
     
-    if (proxyUrl.startsWith("socks")) return new SocksProxyAgent(proxyUrl);
-    if (proxyUrl.startsWith("https")) return new HttpsProxyAgent(proxyUrl);
-    if (proxyUrl.startsWith("http")) return new HttpProxyAgent(proxyUrl);
-    return null;
+    if (proxyUrl.startsWith("socks")) proxy = new SocksProxyAgent(proxyUrl);
+    else if (proxyUrl.startsWith("https")) proxy = new HttpsProxyAgent(proxyUrl);
+    else if (proxyUrl.startsWith("http")) proxy = new HttpProxyAgent(proxyUrl);
+
+    proxy.USINGSCRAPED = proxyUrl;
+
+    return proxy;
 }