puppyproxy 1.0.1 → 1.1.0

package/README.md

@@ -166,7 +166,7 @@ log.highlight("TESTING: ProxyRequest");
 
 //proxyRequest is a wrapper that allows you to use a more consistent proxy for multiple requests, with auto-rotation when it fails
 const proxyRequest = pp.createProxyRequest(ipUrl, {
-    autoNewAgent: true, // auto rotate the proxy when a request fails. if this is false if you really need just one proxy you can manage yourself
+    autoNewAgent: true, // auto rotate the proxy when a request fails. set this to false if you really need just one proxy you can manage yourself
 });
 
 log.bold("----- proxyRequest.fetch");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "puppyproxy",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "Node.js library for managing, scraping, and using proxies with fetch and WebSocket support",
   "type": "module",
   "types": "./src/index.d.ts",
@@ -29,12 +29,14 @@
   "dependencies": {
     "http-proxy-agent": "^7.0.2",
     "https-proxy-agent": "^7.0.6",
+    "node-fetch": "^3.3.2",
     "p-limit": "^7.2.0",
     "puppyid": "^1.1.0",
     "puppylog": "^1.2.0",
     "puppymisc": "^1.1.1",
     "puppyscrambled": "^1.1.0",
     "socks-proxy-agent": "^8.0.5",
-    "sqlite3": "^5.1.7"
+    "sqlite3": "^5.1.7",
+    "ws": "^8.19.0"
   }
 }

package/src/providers/Scraper.js

@@ -8,11 +8,12 @@ import log from 'puppylog';
 import id from 'puppyid';
 import { getLastSavedTimestamp } from 'puppymisc';
 import scrambled from 'puppyscrambled';
+import { createBasicAgent } from '../utils.js';
 
 export default class Scraper {
     constructor(config, proxyDB) {
         this.config = config;
-        this.proxyDB = proxyDB; // Reference to ProxyDatabase instance
+        this.proxyDB = proxyDB;
         this.hosts = [];
         this.hostUsage = {};
         
@@ -21,14 +22,7 @@ export default class Scraper {
         this.outputPath = path.join(this.storeDir, 'scraped_proxies.json');
     }
 
-    createAgent(proxyUrl) {
-        if (proxyUrl.startsWith('socks')) return new SocksProxyAgent(proxyUrl);
-        if (proxyUrl.startsWith('http:')) return new HttpProxyAgent(proxyUrl);
-        if (proxyUrl.startsWith('https:')) return new HttpsProxyAgent(proxyUrl);
-        return null;
-    }
-
-    genHostsScraper() {
+    loadScrapedProxies() {
         try {
             if (!fs.existsSync(this.outputPath)) return;
             const data = fs.readFileSync(this.outputPath, 'utf8');
@@ -48,11 +42,6 @@ export default class Scraper {
         }
     }
 
-    // Alias for init
-    async loadScrapedProxies() {
-        this.genHostsScraper();
-    }
-
     getHostAny(url = '') {
         if (!this.hosts || this.hosts.length === 0) {
             log.error(id, "No hosts available from scraper! IP would be exposed!");
@@ -95,7 +84,7 @@ export default class Scraper {
             const socksProxy = this.getHost(url, type);
             if (!socksProxy) return null;
 
-            const agent = this.createAgent(socksProxy);
+            const agent = createBasicAgent(socksProxy);
             agent.USINGSCRAPED = socksProxy;
             return agent;
         }
@@ -127,7 +116,7 @@ export default class Scraper {
                     worker.terminate();
                     // The worker returns raw proxies, we process them into DB
                     await this.proxyDB.dealWithProxies(message, this.outputPath);
-                    this.genHostsScraper(); // Reload memory
+                    this.loadScrapedProxies(); // Reload memory
                     resolve();
                 });
 

package/src/workers/collector.js

@@ -15,8 +15,6 @@ process.on('unhandledRejection', () => {});
 process.on('uncaughtException', () => {});
 
 let config = {};
-const wsUrl = 'wss://shellshock.io/matchmaker/';
-const httpUrl = 'https://st1.lumidiagames.com';
 
 /**
  * Fetches and parses proxy lists from multiple public repositories
@@ -75,12 +73,14 @@ async function checkProxy(socksOrHttpProxy) {
 
     const maxTimeAllowed = config.scraper.timeoutMax; //in seconds, lower for less proxies but higher quality
 
-    let method = "http"; // 'ws' or 'http' , ws requires LS dev to be running
-    // log.dim(socksOrHttpProxy)
+    let method = config.scraper.method || "http"; // 'ws' or 'http'
+    const wsUrl = config.scraper.wsUrl || 'wss://shellshock.io/matchmaker/';
+    const httpUrl = config.scraper.httpUrl || 'https://st1.lumidiagames.com';
+    // log.dim(socksOrHttpProxy, "checking...", method, "method", wsUrl, httpUrl);
 
-    if (socksOrHttpProxy.startsWith("http")) {
-        method = "http";
-    };
+    // if (socksOrHttpProxy.startsWith("http")) {
+    //     method = "http";
+    // };
 
     const timeoutMs = (config.scraper.timeoutPerProxy || 15) * 1e3;
 
@@ -123,22 +123,7 @@ async function checkProxy(socksOrHttpProxy) {
     } else return new Promise(async (resolve) => {
         try {
             const agent = createBasicAgent(socksOrHttpProxy);
-            const options = { agent, 
-                headers: {
-                    'accept-encoding': 'gzip, deflate, br, zstd',
-                    'accept-language': 'en-US,en;q=0.9',
-                    'cache-control': 'no-cache',
-                    'connection': 'Upgrade',
-                    'host': 'shellshock.io',
-                    'origin': 'https://shellshock.io',
-                    'pragma': 'no-cache',
-                    'sec-websocket-extensions': 'permessage-deflate; client_max_window_bits',
-                    // 'sec-websocket-key': generateWebSocketKey(),
-                    // 'sec-websocket-version': '13',
-                    // 'upgrade': 'websocket',
-                    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'
-                }
-            };
+            const options = { agent };
     
             let func = ()=>{};
 
@@ -156,42 +141,58 @@ async function checkProxy(socksOrHttpProxy) {
             let timeStart = Date.now();
     
             ws.on('open', () => {
-                config.scraper.verbose && log.magenta(`Connected using proxy: ${socksProxy}`);
-                ws.send('{ "command": "regionList" }');
+                config.scraper.verbose && log.magenta(`Connected using proxy: ${socksOrHttpProxy}`);
+
+
+                clearTimeout(timeout);
+                let timeTaken = (Date.now() - timeStart)/1000;
+                // ips.add(ip.ip);
+                ws.close();
+                if (timeTaken > maxTimeAllowed) {
+                    config.scraper.verbose && log.orange(`Result returned for proxy: ${socksOrHttpProxy} in [${timeTaken.toFixed(2)}s]`);
+                    socksOrHttpProxy = null;
+                } else {
+                    config.scraper.verbose && log.success(`Result returned for proxy: ${socksOrHttpProxy} in [${timeTaken.toFixed(2)}s]`);
+                };
+                resolve(socksOrHttpProxy);
+
+
+                // ws.send('{ "command": "regionList" }');
             });
     
             ws.on('message', (data) => {
-                clearTimeout(timeout);
-                const msg = JSON.parse(data.toString());
-                if (msg.command == "regionList") {
-                    let timeTaken = (Date.now() - timeStart)/1000;
-                    // ips.add(ip.ip);
-                    ws.close();
-                    if (timeTaken > maxTimeAllowed) {
-                        config.scraper.verbose && log.orange(`Result returned for proxy: ${socksOrHttpProxy} in [${timeTaken.toFixed(2)}s]`);
-                        socksOrHttpProxy = null;
-                    } else {
-                        config.scraper.verbose && log.success(`Result returned for proxy: ${socksOrHttpProxy} in [${timeTaken.toFixed(2)}s]`);
-                    };
-                    resolve(socksOrHttpProxy);
-                } else {
-                    // log.dim(msg)
-                }
+                // clearTimeout(timeout);
+                // const msg = JSON.parse(data.toString());
+                // if (msg.command == "regionList") {
+                //     let timeTaken = (Date.now() - timeStart)/1000;
+                //     // ips.add(ip.ip);
+                //     ws.close();
+                //     if (timeTaken > maxTimeAllowed) {
+                //         config.scraper.verbose && log.orange(`Result returned for proxy: ${socksOrHttpProxy} in [${timeTaken.toFixed(2)}s]`);
+                //         socksOrHttpProxy = null;
+                //     } else {
+                //         config.scraper.verbose && log.success(`Result returned for proxy: ${socksOrHttpProxy} in [${timeTaken.toFixed(2)}s]`);
+                //     };
+                //     resolve(socksOrHttpProxy);
+                // } else {
+                //     // log.dim(msg)
+                // }
             });
     
             ws.on('error', (err) => {
                 clearTimeout(timeout);
-                // log.dim(`Error with proxy ${socksProxy}:`, err.message);
+                // log.dim(`Error with proxy ${socksOrHttpProxy}:`, err.message);
                 // log.error(err);
                 resolve(null);
             });
     
-            ws.on('close', () => {
+            ws.on('close', (event) => {
                 clearTimeout(timeout);
+                // console.log(`WebSocket closed with code: ${event.code}, reason: ${event.reason}`);
                 resolve(null);
             });
         } catch (error) {
-            // log.error("Error checking proxy:", error);
+            console.error("Error checking proxy:", error);
             resolve(null);
         };
     });
@@ -199,6 +200,7 @@ async function checkProxy(socksOrHttpProxy) {
 
 async function getProxies() {
     log.dim("Fetching all proxies from sources...");
+    config.scraper.verbose && console.log(config.scraper);
 
     let allProxies = await fetchAllProxies();