puppyproxy 0.0.0-security → 1.0.0

package/src/workers/collector.js

@@ -0,0 +1,311 @@
+import { parentPort } from 'worker_threads';
+import fetch from 'node-fetch';
+import WebSocket from 'ws';
+import pLimit from 'p-limit';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import { PROXY_SOURCES } from '../constants.js';
+import scrambled from 'puppyscrambled';
+import log from 'puppylog';
+import { createBasicAgent } from '../utils.js';
+
+//fuck it
+process.on('unhandledRejection', () => {});
+process.on('uncaughtException', () => {});
+
+let config = {};
+const wsUrl = 'wss://shellshock.io/matchmaker/';
+const httpUrl = 'https://st1.lumidiagames.com';
+
+/**
+ * Fetches and parses proxy lists from multiple public repositories
+ */
+async function fetchAllProxies() {
+    let proxyList = [];
+
+    async function parseTraditionalList(url, kind) {
+        try {
+            const response = await fetch(`${url}?v=${Date.now()}`);
+            const txt = await response.text();
+            const proxies = txt.trim()
+                .split("\n")
+                .map(p => p.trim())
+                .filter(p => p.length > 0)
+                .map(p => p.includes("://") ? p : `${kind}${p}`);
+            
+            proxyList.push(...proxies);
+        } catch (err) {
+            // Silently fail individual lists to keep the worker moving
+        }
+    }
+
+    const tasks = [];
+    
+    // Add SOCKS5 sources
+    if (config.scraper.proxyTypes.includes("socks5")) PROXY_SOURCES.socks5.forEach(url => tasks.push(parseTraditionalList(url, "socks5://")));
+    // Add SOCKS4 sources
+    if (config.scraper.proxyTypes.includes("socks4")) PROXY_SOURCES.socks4.forEach(url => tasks.push(parseTraditionalList(url, "socks4://")));
+    // Add HTTP/S sources
+    if (config.scraper.proxyTypes.includes("http")) PROXY_SOURCES.http.forEach(url => tasks.push(parseTraditionalList(url, "http://")));
+    if (config.scraper.proxyTypes.includes("https")) PROXY_SOURCES.https.forEach(url => tasks.push(parseTraditionalList(url, "https://")));
+
+    await Promise.allSettled(tasks);
+
+    let uniqueProxies = [...new Set(scrambled.shuffleArray(proxyList))];
+    
+    const limit = config.scraper?.limitProxies || 40000;
+    if (uniqueProxies.length > limit) {
+        uniqueProxies = uniqueProxies.slice(0, limit);
+    }
+
+    return uniqueProxies;
+}
+
+/**
+ * Verifies if a proxy is alive and meets the latency requirements
+ */
+async function checkProxy(socksOrHttpProxy) {
+    const proxyRegex = /^(socks5|socks4|socks5h|http|https):\/\/(?:[^:@\s]+(?::[^:@\s]*)?@)?\d{1,3}(?:\.\d{1,3}){3}:\d{2,5}$/i;
+
+    if (!proxyRegex.test(socksOrHttpProxy)) {
+        // log.warning("Invalid proxy format:", socksOrHttpProxy);
+        return false;
+    }
+
+    const maxTimeAllowed = config.scraper.timeoutMax; //in seconds, lower for less proxies but higher quality
+
+    let method = "http"; // 'ws' or 'http' , ws requires LS dev to be running
+    // log.dim(socksOrHttpProxy)
+
+    if (socksOrHttpProxy.startsWith("http")) {
+        method = "http";
+    };
+
+    const timeoutMs = (config.scraper.timeoutPerProxy || 15) * 1e3;
+
+    if (method === "http") {
+        return new Promise(async (resolve) => {
+            try {
+                let agent = createBasicAgent(socksOrHttpProxy);
+                let timeStart = Date.now();
+    
+                const controller = new AbortController();
+                setTimeout(() => controller.abort(), timeoutMs);
+
+                const response = await fetch(httpUrl, {
+                    method: 'HEAD',
+                    agent,
+                    signal: controller.signal
+                });
+
+                let timeTaken = (Date.now() - timeStart)/1000;
+    
+                if (response.status === 200) {
+                    if (timeTaken > maxTimeAllowed) {
+                        config.scraper.verbose && log.orange(`HTTP response returned for proxy: ${socksOrHttpProxy} in [${timeTaken.toFixed(2)}s]`);
+                        socksOrHttpProxy = null;
+                    } else {
+                        config.scraper.verbose && log.success(`HTTP response returned for proxy: ${socksOrHttpProxy} in [${timeTaken.toFixed(2)}s]`);
+                    };
+                    resolve(socksOrHttpProxy);
+                } else {
+                    resolve(null);
+                };
+            } catch (error) {
+                if (error.code === 'ECONNRESET') {
+                    return resolve(null);
+                }
+                // console.error("Error checking proxy:", error);
+                resolve(null);
+            };
+        });
+    } else return new Promise(async (resolve) => {
+        try {
+            const agent = createBasicAgent(socksOrHttpProxy);
+            const options = { agent, 
+                headers: {
+                    'accept-encoding': 'gzip, deflate, br, zstd',
+                    'accept-language': 'en-US,en;q=0.9',
+                    'cache-control': 'no-cache',
+                    'connection': 'Upgrade',
+                    'host': 'shellshock.io',
+                    'origin': 'https://shellshock.io',
+                    'pragma': 'no-cache',
+                    'sec-websocket-extensions': 'permessage-deflate; client_max_window_bits',
+                    // 'sec-websocket-key': generateWebSocketKey(),
+                    // 'sec-websocket-version': '13',
+                    // 'upgrade': 'websocket',
+                    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'
+                }
+            };
+    
+            let func = ()=>{};
+
+            let timeout = setTimeout(() => {
+                func();
+                resolve(null);
+            }, timeoutMs);
+
+            // await new Promise(resolve => setTimeout(resolve, scrambled.getRandomInt(0, 15e3)));
+    
+            const ws = new WebSocket(wsUrl, options);
+
+            func = ()=>ws.close();
+
+            let timeStart = Date.now();
+    
+            ws.on('open', () => {
+                config.scraper.verbose && log.magenta(`Connected using proxy: ${socksProxy}`);
+                ws.send('{ "command": "regionList" }');
+            });
+    
+            ws.on('message', (data) => {
+                clearTimeout(timeout);
+                const msg = JSON.parse(data.toString());
+                if (msg.command == "regionList") {
+                    let timeTaken = (Date.now() - timeStart)/1000;
+                    // ips.add(ip.ip);
+                    ws.close();
+                    if (timeTaken > maxTimeAllowed) {
+                        config.scraper.verbose && log.orange(`Result returned for proxy: ${socksOrHttpProxy} in [${timeTaken.toFixed(2)}s]`);
+                        socksOrHttpProxy = null;
+                    } else {
+                        config.scraper.verbose && log.success(`Result returned for proxy: ${socksOrHttpProxy} in [${timeTaken.toFixed(2)}s]`);
+                    };
+                    resolve(socksOrHttpProxy);
+                } else {
+                    // log.dim(msg)
+                }
+            });
+    
+            ws.on('error', (err) => {
+                clearTimeout(timeout);
+                // log.dim(`Error with proxy ${socksProxy}:`, err.message);
+                // log.error(err);
+                resolve(null);
+            });
+    
+            ws.on('close', () => {
+                clearTimeout(timeout);
+                resolve(null);
+            });
+        } catch (error) {
+            // log.error("Error checking proxy:", error);
+            resolve(null);
+        };
+    });
+};
+
+async function getProxies() {
+    log.dim("Fetching all proxies from sources...");
+
+    let allProxies = await fetchAllProxies();
+
+    allProxies = scrambled.shuffleArray(allProxies);
+
+    if (allProxies.length === 0) {
+        log.dim("No proxies to test.");
+        return;
+    };
+
+    const checkCount = config.scraper?.maxProxiesToCheck || 4000;
+    log.dim(`Got ${allProxies.length} proxies. Testing up to ${checkCount} proxies of each type...`);
+    let time = Date.now();
+
+    async function testProxiesSpaced(type, proxies) {
+        const limit = pLimit(250);
+        let completed = 0;
+        let success = 0;
+        const total = proxies.length;
+        const start = Date.now();
+
+        const interval = setInterval(() => {
+            const elapsed = (Date.now() - start) / 1000;
+            const rate = completed / elapsed;
+            const remaining = total - completed;
+            const eta = rate > 0 ? remaining / rate : Infinity;
+
+            config.scraper.logStatus && log.boldGreen(
+                `✅ Checked ${completed}/${total} proxies ` +
+                `(${((completed / total) * 100).toFixed(1)}%) ` +
+                `| 🟢 Success: ${success} ` +
+                `| Elapsed: ${elapsed.toFixed(1)}s ` +
+                `| ETA: ${eta.toFixed(1)}s`
+            );
+        }, 5000);
+
+        const results = await Promise.all(
+            proxies.map(proxy =>
+                limit(() =>
+                    checkProxy(proxy.replaceAll("\r", ""))
+                        .then(result => {
+                            if (result) success++;
+                            return result;
+                        })
+                        .finally(() => completed++)
+                )
+            )
+        );
+
+        clearInterval(interval);
+        const duration = ((Date.now() - start) / 1000).toFixed(1);
+        log.dim(`🏁 Finished checking all ${total} ${type} proxies in ${duration}s. 🟢 Success: ${success}`);
+
+        return results;
+    }
+
+    const socksProxiesOnly = allProxies.filter(p => p.startsWith("socks")).slice(0, checkCount);
+    const httpProxiesOnly  = allProxies.filter(p => p.startsWith("http")).slice(0, checkCount);
+
+    log.dim(`Will test ${socksProxiesOnly.length} SOCKS proxies and ${httpProxiesOnly.length} HTTP proxies. ATTENTION! This can take AGES!! Like up to 15 minutes or more depending on the amount of proxies and their quality.`);
+    log.dim("doing socks.", socksProxiesOnly.length);
+    const socksResults = await testProxiesSpaced("socks", socksProxiesOnly);
+    log.dim("finished socks. doing http.", httpProxiesOnly.length);
+    const httpResults = await testProxiesSpaced("http", httpProxiesOnly);
+    log.dim("finished http. continuing...");
+
+    const workingSocks = socksResults.filter(Boolean);
+    const workingHttp  = httpResults.filter(Boolean);
+
+    log.dim(`✅ SOCKS proxies working: ${workingSocks.length} / ${socksProxiesOnly.length}`);
+    log.dim(`✅ HTTP(S) proxies working: ${workingHttp.length} / ${httpProxiesOnly.length}`);
+
+    let workingProxies = [...workingSocks, ...workingHttp];
+    log.dim(`\nTesting complete. Found ${workingProxies.length} working proxies. Total: ${allProxies.length}. ${((workingProxies.length/allProxies.length)*100).toFixed(2)}%`);
+    log.dim(`\nTime taken: ${(Date.now() - time)/1000}s`);
+    // log.dim("Working proxies:", workingProxies);
+    // log.dim("Unique IPs:", Array.from(ips));
+
+    workingProxies = scrambled.shuffleArray(workingProxies);
+
+    allProxies = null;
+    // ips.clear();
+
+    return workingProxies;
+};
+
+/**
+ * Main Thread Communication
+ */
+if (parentPort) {
+    parentPort.on('message', async (msg) => {
+        const [cmd, payload] = msg;
+        
+        if (cmd === "deploy") {
+            config = payload.config;
+            
+            log.dim("Worker: Started.");
+            // const allProxies = await fetchAllProxies();
+            // log.dim(`Worker: Testing ${allProxies.length} proxies...`);
+
+            // const limit = pLimit(250); // Concurrent limit to prevent OS socket exhaustion
+            // const results = await Promise.all(
+            //     allProxies.map(p => limit(() => checkProxy(p.replace("\r", ""))))
+            // );
+
+            // const workingProxies = results.filter(Boolean);
+            parentPort.postMessage(await getProxies());
+        }
+    });
+}