puppyproxy 0.0.0-security → 1.0.0

package/README.md

@@ -1 +1,261 @@
-WIP, coming soon
+# puppyproxy
+
+[On npm @ https://www.npmjs.com/package/puppyproxy](https://www.npmjs.com/package/puppyproxy)
+
+[And GitHub @ https://github.com/onlypuppy7/puppyproxy](https://github.com/onlypuppy7/puppyproxy)
+
+Robust and reliable free proxy scraper and request wrapper, ported to npm. Another useless file taken out of my projects!
+
+## Install
+`npm install puppyproxy`
+
+## Usage
+```js
+import PuppyProxy from 'puppyproxy';
+import log from 'puppylog'; //another one of my packages
+import { wait } from 'puppymisc'; //another one of my packages!
+
+const ipUrl = 'http://ifconfig.me/ip';
+const wsUrl = 'wss://ws.postman-echo.com/raw';
+
+const config = {
+    // storeDir: './store', // optional, defaults to platform specific data dir
+    ipvanish: {
+        // use: true,
+        // creds: { username: 'abc', password: '123' },
+    },
+    scraper: {
+        use: true,
+        timeBetweenScrapes: 45 * 60e3,
+        maxProxiesToCheck: 5000, // the higher this is, the longer it takes, but the more proxies you get
+        limitProxies: 500000,
+        timeoutPerProxy: 5,
+        timeoutMax: 5,
+        verbose: true, // for logging purposes
+        logStatus: true, // shows a convenient summary every so often
+        proxyTypes: ['socks4', 'socks5'], //['socks4', 'socks5', 'http', 'https']
+
+        timeoutFetch: 10000, // ms
+        timeoutWs: 7000,
+        maxRetries: 10,
+    },
+    LOG: { connect: true, error: true } // for logging purposes
+};
+
+const pp = new PuppyProxy(config);
+
+log.highlight("TESTING: runCollector");
+
+await pp.init(true); // true forces run the collector
+
+log.highlight("TESTING: RequestClient");
+
+log.bold("----- pp.request.fetch")
+
+//fetch with inbuilt resilient retrying in case of error. uses a new proxy from the pool each time
+const resRequestFetch = await pp.request.fetch(ipUrl);
+log.success("res from request fetch", await resRequestFetch.text());
+
+log.bold("----- pp.request.fetchBasic")
+
+//more basic fetch that just forwards to you the req object with a proxy agent set
+try {
+    const resFetchBasic = await (pp.request.fetchBasic(ipUrl));
+    if (resFetchBasic?.ok) {
+        log.success("res from basic fetch", await resFetchBasic.text());
+    } else {
+        //this may fail because free proxies are unreliable, and it only tries once
+        log.error("basic fetch failed", resFetchBasic.status);
+    }
+} catch (error) {
+    log.error("basic fetch error, which is sort of expected. create your own catching logic.", error.message);
+}
+
+log.bold("----- pp.request.ws")
+
+//ws with inbuilt resilient retrying in case of error. uses a new proxy from the pool each time
+const testWs = async (wsHandler) => {
+    return new Promise(async (resolve, reject) => {
+        await wsHandler(
+            wsUrl,
+            {
+                // preferredAgent: 'none' // 'ipvanish', 'scraper', or 'none'
+            },
+            // open
+            (ws) => {
+                const testMessage = 'echo, test message';
+                log.success(`Connected to echo server, sending test message ${testMessage}`);
+                ws.send(JSON.stringify(testMessage));
+            },
+            // message
+            (ws, event) => {
+                try {
+                    const data = JSON.parse(event.data);
+                    log.success('Received JSON:', data);
+                } catch {
+                    log.success('Received text:', event.data);
+                }
+
+                // close after we get responses
+                ws.close();
+                wait(500).then(resolve);
+            },
+            // close
+            (ws, event) => {
+                log.italic(`WebSocket closed with code: ${event.code}, reason: ${event.reason}`);
+            },
+            // error
+            (ws, error) => {
+                // reject(error);
+            }
+        );
+        resolve();
+    });
+}
+
+try {
+    await testWs(pp.request.ws.bind(pp.request));
+} catch (error) {
+    log.error("WebSocket connection failed:", error.message);
+};
+
+log.bold("----- pp.request.wsBasic");
+
+const testWsBasic = async (wsBasicHandler) => {
+    try {
+        await new Promise(async (resolve, reject) => {
+            const ws = wsBasicHandler(wsUrl, {});
+
+            ws.onopen = (event) => {
+                const testMessage = 'echo, test message via wsBasic';
+                log.success(`Connected to echo server via wsBasic, sending test message ${testMessage}`);
+                ws.send(JSON.stringify(testMessage));
+            };
+
+            ws.onmessage = (event) => {
+                try {
+                    const data = JSON.parse(event.data);
+                    log.success('wsBasic Received JSON:', data);
+                } catch {
+                    log.success('wsBasic Received text:', event.data);
+                }
+
+                ws.close();
+                wait(500).then(resolve);
+            };
+
+            ws.onclose = (event) => {
+                log.italic(`wsBasic WebSocket closed with code: ${event.code}, reason: ${event.reason}`);
+            };
+
+            ws.onerror = (error) => {
+                log.error("wsBasic WebSocket error, which is sort of expected. create your own catching logic.", error.message);
+                reject(error);
+            };
+        });
+    } catch (error) {
+        // log.error("proxyRequest wsBasic WebSocket connection failed:", error.message);
+    };
+};
+
+//similar basic method like pp.request.fetchBasic exists for pp.request.wsBasic if you want less resilience and more control.
+
+await testWsBasic(pp.request.wsBasic.bind(pp.request));
+
+log.highlight("TESTING: ProxyRequest");
+
+//proxyRequest is a wrapper that allows you to use a more consistent proxy for multiple requests, with auto-rotation when it fails
+const proxyRequest = pp.createProxyRequest(ipUrl, {
+    autoNewAgent: true, // auto rotate the proxy when a request fails. if this is false if you really need just one proxy you can manage yourself
+});
+
+log.bold("----- proxyRequest.fetch");
+
+//all methods are similar to pp.request but use the same proxy until it fails
+const resProxyRequestFetch = await proxyRequest.fetch(); //url can be passed here, but it already was set in constructor
+log.success("res from proxyRequest fetch", await resProxyRequestFetch.text());
+
+log.bold("----- proxyRequest.fetchBasic");
+
+const testProxyRequestFetchBasic = async () => {
+    try {
+        const resProxyRequestFetchBasic = await (proxyRequest.fetchBasic());
+        if (resProxyRequestFetchBasic?.ok) {
+            log.success("res from proxyRequest basic fetch", await resProxyRequestFetchBasic.text());
+        } else {
+            log.error("proxyRequest basic fetch failed", resProxyRequestFetchBasic.status);
+        }
+    } catch (error) {
+        log.error("proxyRequest basic fetch error, which is sort of expected. create your own catching logic.", error.message);
+    }
+};
+
+await testProxyRequestFetchBasic();
+//try again to see it use the same proxy
+await testProxyRequestFetchBasic();
+
+log.bold("----- proxyRequest.newAgent");
+
+//you can manually rotate the proxy too
+proxyRequest.newAgent();
+
+await testProxyRequestFetchBasic();
+
+log.bold("----- proxyRequest.ws");
+
+try {
+    await testWs(proxyRequest.ws.bind(proxyRequest));
+} catch (error) {
+    log.error("proxyRequest WebSocket connection failed:", error.message);
+}
+
+log.bold("----- proxyRequest.wsBasic");
+
+await testWsBasic(proxyRequest.wsBasic.bind(proxyRequest));
+
+log.highlight("TESTING: createGlobalPatch");
+
+pp.createGlobalPatch();
+
+log.bold("----- global fetch")
+
+const resGlobalFetch = await fetch(ipUrl);
+log.success("res from global fetch", await resGlobalFetch.text());
+
+//note: i dont think you can override node-fetch. sad!
+
+log.bold("----- global WebSocket")
+
+try {
+    await new Promise((resolve, reject) => {
+        const ws = new WebSocket(wsUrl);
+        ws.onopen = (event) => {
+            const testMessage = 'echo, test message via global WebSocket';
+            log.success(`Connected to echo server via global WebSocket, sending test message ${testMessage}`);
+            ws.send(JSON.stringify(testMessage));
+        };
+
+        ws.onmessage = (event) => {
+            try {
+                const data = JSON.parse(event.data);
+                log.success('global WebSocket Received JSON:', data);
+            } catch {
+                log.success('global WebSocket Received text:', event.data);
+            }
+            ws.close();
+            wait(500).then(resolve);
+        };
+
+        ws.onclose = (event) => {
+            log.italic(`global WebSocket closed with code: ${event.code}, reason: ${event.reason}`);
+        };
+
+        ws.onerror = (error) => {
+            log.error("global WebSocket error, which is sort of expected. create your own catching logic.", error.message);
+            reject(error);
+        };
+    });
+} catch (error) {
+    log.error("global WebSocket connection failed:", error.message);
+}
+```

package/package.json

@@ -1,6 +1,29 @@
-{
-  "name": "puppyproxy",
-  "description": "coming soon!",
-  "version": "0.0.0-security",
-  "files": ["package.json", "README.md"]
-}
+{
+  "name": "puppyproxy",
+  "version": "1.0.0",
+  "description": "",
+  "type": "module",
+  "types": "./src/index.d.ts",
+  "main": "./src/index.js",
+  "exports": {
+    ".": "./src/index.js"
+  },
+  "keywords": [],
+  "scripts": {
+    "test": "node src/tests/test.js",
+    "build": "node src/tests/build.js"
+  },
+  "author": "onlypuppy7",
+  "license": "MIT",
+  "dependencies": {
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.6",
+    "p-limit": "^7.2.0",
+    "puppyid": "^1.1.0",
+    "puppylog": "^1.2.0",
+    "puppymisc": "^1.1.1",
+    "puppyscrambled": "^1.1.0",
+    "socks-proxy-agent": "^8.0.5",
+    "sqlite3": "^5.1.7"
+  }
+}

package/src/PuppyProxy.js

@@ -0,0 +1,131 @@
+import IPVanish from './providers/IPVanish.js';
+import Scraper from './providers/Scraper.js';
+import ProxyDatabase from './database/ProxyDatabase.js';
+import RequestClient from './network/RequestClient.js';
+import log from 'puppylog';
+import id from 'puppyid';
+import { originals } from './constants.js';
+import os from 'os';
+import path from 'path';
+
+export default class PuppyProxy {
+    /**
+     * @param {Object} config - The configuration object
+     */
+    constructor(config) {
+        this.config = config;
+        this.config.storeDir = this.config.storeDir || this.resolveDataDir();
+        console.log("Using data store directory:", this.config.storeDir);
+        
+        // Initialize Modules
+        this.proxyDB = new ProxyDatabase(this.config);
+        
+        this.ipvanish = new IPVanish(this.config);
+        this.scraper = new Scraper(this.config, this.proxyDB);
+        
+        // Request Client needs access to this instance to get agents
+        this.request = new RequestClient(this, this.config);
+    }
+    
+    resolveDataDir() {
+        const appName = 'puppyproxy-data';
+        switch (process.platform) {
+            case 'win32':
+                return path.join(process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local'), appName);
+            case 'darwin':
+                return path.join(os.homedir(), 'Library', 'Application Support', appName);
+            default: // Linux and others
+                return path.join(process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share'), appName);
+        }
+    }
+
+    /**
+     * Initialize the PuppyProxy system
+     * @param {boolean} forceCollector - Whether to force run the proxy collector
+     */
+    async init(forceCollector = false) {
+        await this.proxyDB.init();
+        
+        // Load scraped proxies into memory
+        await this.scraper.runCollector(forceCollector);
+        await this.scraper.loadScrapedProxies();
+    }
+
+    /**
+     * Centralized method to get an agent based on preference
+     * @param {string} url 
+     * @param {string} preferredAgent - 'ipvanish', 'scraper', or 'none'
+     */
+    getAgent(url, preferredAgent = null) {
+        const agents = {
+            ipvanish: this.ipvanish.getAgent(url),
+            scraper: this.scraper.getAgent(url),
+            none: null
+        };
+
+        // Determine which agent to use based on config and preference
+        let use = null;
+        
+        if (preferredAgent && agents[preferredAgent]) {
+            use = preferredAgent;
+        } else if (preferredAgent !== "none") {
+            if (this.config.ipvanish?.use) use = "ipvanish";
+            else if (this.config.scraper?.use) use = "scraper";
+        }
+
+        const agent = use ? agents[use] : null;
+
+        // Logging
+        if (agent) {
+            if (this.config.LOG?.connect) {
+                log.magenta(id, `Attempting to connect to ${url} via ${use}: ${agent.USINGSCRAPED || 'internal'}`);
+            }
+        } else if (preferredAgent !== "none" && (this.config.ipvanish?.use || this.config.scraper?.use)) {
+            const errorMsg = `Error!! Wanted proxy but none returned for connect to ${url} (Pref: ${preferredAgent})`;
+            log.error(id, errorMsg);
+            throw new Error(errorMsg);
+        } else {
+            if (this.config.LOG?.connectNoProxy) {
+                log.muted(id, `Attempting to connect to ${url} with no proxy`);
+            }
+        }
+
+        return agent;
+    }
+
+    createProxyRequest(url, options = {}) {
+        return this.request.createProxyRequest(url, options);
+    }
+
+    /**
+     * Trigger the proxy collection worker
+     */
+    async collectProxies(force = false) {
+        return this.scraper.runCollector(force);
+    }
+
+    createGlobalPatch(fetchOverride = this.request.fetch.bind(this.request), wsOverride = this.request.wsBasic.bind(this.request), fetchRule = () => true, wsRule = () => true) {
+        console.log(!!originals.fetch, !!originals.WebSocket, "Creating global fetch/WebSocket patch for PuppyProxy");
+
+        //override global fetch and WebSocket to use PuppyProxy
+
+        if (fetchOverride) {
+            const newFetch = (url, options = {}) => {
+                if (!fetchRule(url, options)) {
+                    return originals.fetch(url, options);
+                } else return fetchOverride(url, options);
+            };
+            //global fetch
+            global.fetch = newFetch;
+            //i couldnt work out node-fetch
+        };
+
+        wsOverride && (global.WebSocket = class extends originals.WebSocket {
+            constructor(...args) {
+                if (args.length > 0 && !wsRule(args[0], args[1])) {
+                    return new originals.WebSocket(...args);
+                } else return wsOverride(...args);
+            }
+        });
+    }
+}

package/src/constants.js

@@ -0,0 +1,82 @@
+import fetch from "node-fetch";
+import { WebSocket } from "ws";
+
+export const IPVANISH_HOSTS = [
+    "syd.socks.ipvanish.com",
+    "tor.socks.ipvanish.com",
+    "par.socks.ipvanish.com",
+    "fra.socks.ipvanish.com",
+    "lin.socks.ipvanish.com",
+    "nrt.socks.ipvanish.com",
+    "ams.socks.ipvanish.com",
+    "waw.socks.ipvanish.com",
+    "lis.socks.ipvanish.com",
+    "sin.socks.ipvanish.com",
+    "mad.socks.ipvanish.com",
+    "sto.socks.ipvanish.com",
+    "lon.socks.ipvanish.com",
+    "iad.socks.ipvanish.com",
+    "atl.socks.ipvanish.com",
+    "chi.socks.ipvanish.com",
+    "dal.socks.ipvanish.com",
+    "den.socks.ipvanish.com",
+    "lax.socks.ipvanish.com",
+    "mia.socks.ipvanish.com",
+    "nyc.socks.ipvanish.com",
+    "phx.socks.ipvanish.com",
+    "sea.socks.ipvanish.com"
+];
+
+export const PROXY_SOURCES = {
+    socks5: [
+        "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/socks5.txt",
+        "https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/socks5.txt",
+        "https://github.com/hookzof/socks5_list/raw/refs/heads/master/proxy.txt",
+        "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt",
+        "https://api.openproxylist.xyz/socks5.txt",
+        "https://raw.githubusercontent.com/B4RC0DE-TM/proxy-list/main/SOCKS5.txt",
+        "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks5.txt",
+        // "https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks5.txt", //not updated
+        "https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5_RAW.txt",
+        "https://proxyspace.pro/socks5.txt",
+        // "https://www.proxy-list.download/api/v1/get?type=socks5", //takes ages
+        // "https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/socks5/data.txt", //27% //dead
+        "https://raw.githubusercontent.com/ErcinDedeoglu/proxies/refs/heads/main/proxies/socks5.txt",
+        "https://github.com/vmheaven/VMHeaven-Free-Proxy-Updated/raw/refs/heads/main/socks5.txt",
+        "https://github.com/Vann-Dev/proxy-list/raw/refs/heads/main/proxies/socks5.txt",
+        "https://github.com/ebrasha/abdal-proxy-hub/raw/refs/heads/main/socks5-proxy-list-by-EbraSha.txt",
+        "https://raw.githubusercontent.com/iplocate/free-proxy-list/refs/heads/main/protocols/socks5.txt",
+        "https://github.com/Anonym0usWork1221/Free-Proxies/raw/refs/heads/main/proxy_files/socks5_proxies.txt",
+        "https://github.com/murtaja89/public-proxies/raw/refs/heads/main/proxies_socks5.txt",
+        "https://github.com/Konorze/public-proxies/raw/refs/heads/main/proxies_socks5.txt",
+        // "https://raw.githubusercontent.com/fyvri/fresh-proxy-list/archive/storage/classic/socks5.txt", //like 50k
+    ],
+    socks4: [
+        "https://github.com/TheSpeedX/PROXY-List/raw/refs/heads/master/socks4.txt",
+        "https://github.com/proxifly/free-proxy-list/raw/refs/heads/main/proxies/protocols/socks4/data.txt",
+        "https://github.com/MuRongPIG/Proxy-Master/raw/refs/heads/main/socks4_checked.txt",
+        "https://github.com/vakhov/fresh-proxy-list/raw/refs/heads/master/socks4.txt",
+        "https://github.com/dpangestuw/Free-Proxy/raw/refs/heads/main/socks4_proxies.txt",
+        "https://github.com/ALIILAPRO/Proxy/raw/refs/heads/main/socks4.txt",
+        "https://github.com/vmheaven/VMHeaven-Free-Proxy-Updated/raw/refs/heads/main/socks4.txt",
+        "https://github.com/Vann-Dev/proxy-list/raw/refs/heads/main/proxies/socks4.txt",
+        "https://github.com/ebrasha/abdal-proxy-hub/raw/refs/heads/main/socks4-proxy-list-by-EbraSha.txt",
+        "https://raw.githubusercontent.com/iplocate/free-proxy-list/refs/heads/main/protocols/socks4.txt",
+        "https://github.com/Anonym0usWork1221/Free-Proxies/raw/refs/heads/main/proxy_files/socks4_proxies.txt",
+        // "https://raw.githubusercontent.com/fyvri/fresh-proxy-list/archive/storage/classic/socks4.txt", //too many
+        "https://api.proxyscrape.com/v2/?request=displayproxies&protocol=socks4"
+    ],
+    http: [
+        "https://github.com/dpangestuw/Free-Proxy/raw/refs/heads/main/http_proxies.txt",
+        "https://github.com/vakhov/fresh-proxy-list/raw/refs/heads/master/http.txt",
+    ],
+    https: [
+        "https://github.com/vakhov/fresh-proxy-list/raw/refs/heads/master/https.txt",
+        "https://github.com/javadbazokar/PROXY-List/raw/refs/heads/main/https.txt",
+    ]
+};
+
+export const originals = {
+    fetch,
+    WebSocket,
+};

package/src/database/DatabaseUtils.js

@@ -0,0 +1,9 @@
+import util from 'node:util';
+
+export function assignDBMethods(db) {
+    Object.assign(db, {
+        runQuery: util.promisify(db.run.bind(db)),
+        getOne: util.promisify(db.get.bind(db)),
+        getAll: util.promisify(db.all.bind(db)),
+    });
+}

package/src/database/ProxyDatabase.js

@@ -0,0 +1,77 @@
+import sqlite3 from 'sqlite3';
+import fs from 'node:fs';
+import path from 'path';
+import { assignDBMethods } from './DatabaseUtils.js';
+
+export default class ProxyDatabase {
+    constructor(config) {
+        this.config = config;
+        this.storeDir = config.storeDir || './store';
+        this.dbPath = path.join(this.storeDir, 'proxies.db');
+        this.db = null;
+    }
+
+    async init() {
+        if (!fs.existsSync(this.storeDir)) fs.mkdirSync(this.storeDir, { recursive: true });
+        this.db = new sqlite3.Database(this.dbPath);
+        this.db.configure("busyTimeout", 30000);
+        assignDBMethods(this.db);
+        await this.setupTables();
+    }
+
+    async setupTables() {
+        await this.db.runQuery(`CREATE TABLE IF NOT EXISTS proxies (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            url TEXT UNIQUE NOT NULL,
+            last_worked_at INTEGER DEFAULT 0,
+            time_added INTEGER DEFAULT (strftime('%s', 'now'))
+        )`);
+        
+        await this.db.runQuery(`CREATE TABLE IF NOT EXISTS flags (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            key TEXT UNIQUE NOT NULL,
+            value TEXT NOT NULL
+        )`);
+
+        await this.db.runQuery(`INSERT OR IGNORE INTO flags (key, value) VALUES (?, ?)`, ["current_iteration", 0]);
+    }
+
+    async dealWithProxies(newProxies, outputPath) {
+        console.log("Dealing with proxies:", newProxies.length);
+        
+        let currentIteration = Number((await this.db.getOne(`SELECT value FROM flags WHERE key = ?`, "current_iteration")).value);
+        
+        let existingProxies = await this.db.getAll(`SELECT url FROM proxies`);
+        
+        for (const proxy of newProxies) {
+            let existingProxy = existingProxies.find(p => p.url === proxy);
+            if (!existingProxy) {
+                await this.db.runQuery(`INSERT OR IGNORE INTO proxies (url, last_worked_at) VALUES (?, ?)`, [proxy, currentIteration]);
+            } else {
+                await this.db.runQuery(`UPDATE proxies SET last_worked_at = ? WHERE url = ?`, [currentIteration, proxy]);
+            }
+        }
+
+        await this.db.runQuery(`UPDATE flags SET value = ? WHERE key = ?`, [currentIteration + 1, "current_iteration"]);
+
+        // Select working proxies (worked in last 3 iterations)
+        let decentProxies = await this.db.getAll(`SELECT url FROM proxies WHERE last_worked_at >= ?`, [currentIteration - 3]);
+        let decentProxiesArray = decentProxies.map(p => p.url);
+
+        // Merge extra proxies
+        const extraProxiesPath = path.join(this.storeDir, 'extra_proxies.json');
+        if (!fs.existsSync(extraProxiesPath)) {
+            fs.writeFileSync(extraProxiesPath, JSON.stringify([]), 'utf8');
+        }
+        let extraProxies = JSON.parse(fs.readFileSync(extraProxiesPath, 'utf8'));
+        decentProxiesArray = [...new Set([...decentProxiesArray, ...extraProxies])];
+
+        console.log(`Writing ${decentProxiesArray.length} proxies to ${outputPath}`);
+        
+        try {
+            fs.writeFileSync(outputPath, JSON.stringify(decentProxiesArray), 'utf8');
+        } catch (err) {
+            console.error("Error writing proxies to json:", err.message);
+        }
+    }
+}