puppyproxy 0.0.0-security → 1.0.1

package/src/index.d.ts

@@ -0,0 +1,186 @@
+// index.d.ts
+/// <reference types="node" />
+
+import { Agent } from 'http';
+import { WebSocket as WS } from 'ws';
+
+/**
+ * Configuration for PuppyProxy
+ */
+export interface PuppyProxyConfig {
+    storeDir?: string;
+    ipvanish?: {
+        use?: boolean;
+        creds?: {
+            username: string;
+            password: string;
+        };
+        reducedmode?: boolean;
+    };
+    scraper?: {
+        use?: boolean;
+        timeBetweenScrapes?: number;
+        maxProxiesToCheck?: number;
+        limitProxies?: number;
+        timeoutPerProxy?: number;
+        timeoutMax?: number;
+        verbose?: boolean;
+        logStatus?: boolean;
+        proxyTypes?: Array<'socks4' | 'socks5' | 'http' | 'https'>;
+        timeoutFetch?: number;
+        timeoutWs?: number;
+        maxRetries?: number;
+    };
+    LOG?: {
+        connect?: boolean;
+        connectNoProxy?: boolean;
+        response?: boolean;
+        error?: boolean;
+        minor?: boolean;
+    };
+}
+
+/**
+ * Main PuppyProxy class
+ */
+export default class PuppyProxy {
+    constructor(config: PuppyProxyConfig);
+    
+    config: PuppyProxyConfig;
+    proxyDB: ProxyDatabase;
+    ipvanish: IPVanish;
+    scraper: Scraper;
+    request: RequestClient;
+
+    /**
+     * Initializes the database and runs the proxy collector.
+     * @param forceCollector Whether to force scraping proxies
+     */
+    init(forceCollector?: boolean): Promise<void>;
+
+    /**
+     * Get an HTTP/SOCKS agent for the given URL.
+     * @param url Target URL
+     * @param preferredAgent 'ipvanish' | 'scraper' | 'none'
+     */
+    getAgent(url: string, preferredAgent?: string): Agent | null;
+
+    /**
+     * Create a ProxyRequest instance
+     */
+    createProxyRequest(url: string, options?: ProxyRequestOptions): ProxyRequest;
+
+    /**
+     * Manually run the proxy collector
+     */
+    collectProxies(force?: boolean): Promise<void>;
+
+    /**
+     * Patches global fetch and WebSocket
+     */
+    createGlobalPatch(
+        fetchOverride?: (url: string, options?: any) => Promise<any>,
+        wsOverride?: (url: string, options?: any) => WS,
+        fetchRule?: (url: string, options?: any) => boolean,
+        wsRule?: (url: string, options?: any) => boolean
+    ): void;
+}
+
+/**
+ * Options for ProxyRequest
+ */
+export interface ProxyRequestOptions {
+    AGENT?: Agent;
+    preferredAgent?: 'ipvanish' | 'scraper' | 'none';
+    autoNewAgent?: boolean;
+    maxRetries?: number;
+}
+
+/**
+ * Request client
+ */
+export class RequestClient {
+    constructor(proxyManager: PuppyProxy, config: PuppyProxyConfig);
+    
+    proxyManager: PuppyProxy;
+    config: PuppyProxyConfig;
+    timeoutWs: number;
+    timeoutFetch: number;
+    maxRetries: number;
+
+    fetchBasic(url: string, options?: any): Promise<any>;
+    fetch(url: string, options?: any): Promise<any>;
+    wsBasic(url: string, options?: any): WS;
+    ws(
+        url: string,
+        options?: any,
+        onopen?: (ws: WS, event?: any) => void,
+        onmessage?: (ws: WS, event?: any) => void,
+        onclose?: (ws: WS, event?: any) => void,
+        onerror?: (ws: WS, error?: any) => void
+    ): Promise<WS>;
+
+    createProxyRequest(url: string, options?: ProxyRequestOptions): ProxyRequest;
+}
+
+/**
+ * ProxyRequest for single URL
+ */
+export class ProxyRequest {
+    constructor(requestClient: RequestClient, url: string, options?: ProxyRequestOptions);
+
+    client: RequestClient;
+    url: string;
+    options: ProxyRequestOptions;
+    successes: number;
+
+    success(): void;
+    failure(): void;
+    newAgent(url?: string): Agent | null;
+
+    fetchBasic(url?: string, dataAppend?: any): Promise<any>;
+    fetch(url?: string, dataAppend?: any): Promise<any>;
+    wsBasic(url?: string, dataAppend?: any): WS;
+    ws(
+        url?: string,
+        dataAppend?: any,
+        onopen?: (ws: WS, event?: any) => void,
+        onmessage?: (ws: WS, event?: any) => void,
+        onclose?: (ws: WS, event?: any) => void,
+        onerror?: (ws: WS, error?: any) => void
+    ): Promise<WS>;
+}
+
+/**
+ * Proxy database wrapper
+ */
+export class ProxyDatabase {
+    constructor(config: PuppyProxyConfig);
+
+    config: PuppyProxyConfig;
+    dbPath: string;
+    db: any;
+
+    init(): Promise<void>;
+    dealWithProxies(newProxies: string[], outputPath: string): Promise<void>;
+}
+
+/**
+ * Scraper for external proxies
+ */
+export class Scraper {
+    constructor(config: PuppyProxyConfig, proxyDB: ProxyDatabase);
+
+    runCollector(force?: boolean): Promise<void>;
+    loadScrapedProxies(): Promise<void>;
+    getAgent(url?: string, type?: 'socks' | 'http'): Agent | null;
+}
+
+/**
+ * IPVanish proxy provider
+ */
+export class IPVanish {
+    constructor(config: PuppyProxyConfig);
+
+    getAgent(url: string): Agent | null;
+}

package/src/index.js

@@ -0,0 +1,2 @@
+import PuppyProxy from './PuppyProxy.js';
+export default PuppyProxy;

package/src/network/RequestClient.js

@@ -0,0 +1,284 @@
+import log from 'puppylog';
+import id from 'puppyid';
+import { originals } from '../constants.js';
+
+export class ProxyRequest {
+    constructor(requestClient, url, options = {}) {
+        this.client = requestClient;
+        this.url = url;
+        this.options = options;
+        this.newAgent();
+        this.successes = 0;
+    }
+
+    success() { this.successes++; }
+    failure() { if (this.options.autoNewAgent) this.newAgent(); }
+
+    newAgent(url = this.url) {
+        const agent = this.client.proxyManager.getAgent(url, this.options.preferredAgent);
+        this.options.AGENT = agent;
+        this.successes = 0;
+        return agent;
+    }
+
+    async fetchBasic(url = this.url, dataAppend = {}) {
+        let req = this.client.fetchBasic(url, {
+            ...this.options,
+            ...dataAppend,
+        });
+
+        return new Promise((resolve, reject) => {
+            req.then((response) => {
+                if (response.ok) {
+                    this.success();
+                    resolve(response);
+                } else {
+                    this.failure();
+                    reject(new Error(`Failed to fetchBasic ${url} with status ${response.status}`));
+                }
+            }).catch((error) => {
+                this.failure();
+                reject(error);
+            });
+        });
+    };
+
+    async fetch(url = this.url, dataAppend = {}) {
+        const retry = async () => {
+            try {
+                const response = await (this.client.fetchBasic(url, {
+                    ...this.options,
+                    ...dataAppend
+                }));
+                if (response.ok) {
+                    this.success();
+                    return response;
+                } else {
+                    this.failure();
+                    throw new Error(`Failed to fetch ${url} status ${response.status}`);
+                }
+            } catch (error) {
+                this.failure();
+                throw error;
+            }
+        };
+
+        let retries = this.options.maxRetries || this.client.maxRetries;
+        for (let attempt = 0; attempt < retries; attempt++) {
+            try {
+                return await retry();
+            } catch (error) {
+                this.client.config.LOG?.error && log.error(id, `ProxyRequest fetch attempt ${attempt + 1} of ${retries} failed for ${url}: ${error.message}`);
+                if (attempt === retries - 1) {
+                    throw error;
+                }
+            }
+        }
+    }
+
+    wsBasic(url = this.url, dataAppend = this.options) {
+        return this.client.wsBasic(url, dataAppend);
+    }
+
+    ws(url = this.url, dataAppend = this.options, onopen, onmessage, onclose, onerror) {
+        return this.client.ws(url, dataAppend, onopen, onmessage, onclose, onerror);
+    }
+}
+
+export default class RequestClient {
+    constructor(proxyManager, config) {
+        this.proxyManager = proxyManager;
+        this.config = config;
+        this.timeoutWs = config.timeoutWs || 7000;
+        this.timeoutFetch = config.timeoutFetch || 15000;
+        this.maxRetries = config.maxRetries || 15;
+    }
+    
+    fetchBasic (url, options = {}) {
+        let agent = options.AGENT || this.proxyManager.getAgent(url, options.preferredAgent);
+        delete options.preferredAgent;
+
+        if (agent) {
+            Object.assign(options, { agent });
+        };
+
+        return originals.fetch(url, options);
+    }
+
+    async fetch(url, options = {}) {
+        let attempts = 0;
+        const retry = async () => {
+            attempts++;
+            try {
+                let agent = options.AGENT || this.proxyManager.getAgent(url, options.preferredAgent);
+                const fetchOpts = { ...options, agent: agent || undefined };
+                delete fetchOpts.AGENT;
+                delete fetchOpts.preferredAgent;
+
+                const controller = new AbortController();
+                const timeout = setTimeout(() => controller.abort(), this.timeoutFetch);
+                fetchOpts.signal = controller.signal;
+
+                const response = await originals.fetch(url, fetchOpts);
+                clearTimeout(timeout);
+
+                if (response.ok) {
+                    if(this.config.LOG?.response) console.log(id, `got ok response ${url}`);
+                    return response;
+                } else {
+                    if(this.config.LOG?.error) log.warning(id, `Fetch attempt ${attempts} failed ${url} status ${response.status}`);
+                    if (attempts < this.maxRetries) return retry();
+                    throw new Error(`Failed after ${this.maxRetries} attempts`);
+                }
+            } catch (error) {
+                if(this.config.LOG?.error) log.error(id, `Fetch error ${attempts}: ${error.message}`);
+                if (attempts < this.maxRetries) return retry();
+                throw error;
+            }
+        };
+        return retry();
+    }
+    
+    wsBasic (url, options = {}) {
+        let agent = options.AGENT || this.proxyManager.getAgent(url, options.preferredAgent);
+
+        if (agent) {
+            Object.assign(options, { agent });
+        };
+
+        try {
+            const ws = new originals.WebSocket(url, options);
+
+            setTimeout(() => {
+                if (ws.readyState !== originals.WebSocket.OPEN) {
+                    console.log("wsBasic timeout, closing ws for", url);
+                    ws.close();
+                }
+            }, this.timeoutWs);
+
+            return ws;
+        } catch (error) {
+            console.log("error creating ws for", url);
+
+            throw error;
+        };
+    }
+
+    ws(url, options = {}, onopen, onmessage, onclose, onerror) {
+        return new Promise((resolve, reject) => {
+            let attempts = 0;
+            const retry = async () => {
+                attempts++;
+
+                try {
+                    let agent = options.AGENT || this.proxyManager.getAgent(url, options.preferredAgent);
+
+                    let attempt = attempts;
+
+                    if (agent) {
+                        Object.assign(options, { agent });
+                    };
+
+                    const ws = new originals.WebSocket(url, options);
+                    
+                    ws.destroyCompletely = () => {
+                        if (!ws || ws.readyState === undefined) {
+                            this.config.LOG.error && log.warning(id, "WebSocket is not initialized or invalid");
+                            return;
+                        };
+                        if (ws && ws.readyState !== originals.WebSocket.CLOSED && ws.readyState !== originals.WebSocket.CLOSING) {
+                            try {
+                                ws.onclose = null;
+                                ws.onerror = null;
+                                ws.onmessage = null;
+                                ws.onopen = null;
+                                ws.removeAllListeners();
+                                this.config.LOG.minor && log.pink(id, "destroyed websocket", url, attempt);
+                            } catch (error) {
+                                this.config.LOG.error && log.error(id, "Error in destroyCompletely", error);
+                            };
+                        };
+                    };
+
+                    let opened = false;
+                    let errored = false;
+                    let message = false;
+
+                    setTimeout(() => {
+                        if (!opened) ws.close();
+                    }, this.timeoutWs);
+
+                    ws.onopen = (event) => {
+                        opened = true;
+                        try {
+                            onopen && onopen(ws, event);
+                        } catch (error) {
+                            this.config.LOG.error && log.error(id, "error in onopen", error);
+                        };
+                        resolve(ws);
+                        return;
+                    };
+
+                    ws.onerror = (error) => {
+                        opened = true; errored = true;
+
+                        ws.destroyCompletely();
+
+                        try {
+                            onerror && onerror(ws, error);
+                        } catch (error) {
+                            this.config.LOG.error && log.error(id, "error in onerror", error);
+                        };
+
+                        if (attempts >= this.maxRetries) {
+                            this.config.LOG.error && log.error(id, `Failed to connect to ${url} after ${this.maxRetries} attempts`);
+                        } else {
+                            this.config.LOG.error && log.warning(id, `Retrying connect to ${url} (${attempts}/${this.maxRetries})`);
+                            retry();
+                        };
+                    };
+
+                    ws.onclose = (event) => {
+                        try {
+                            // console.log(id, "ws closed", url, event.code, event.reason, event.wasClean, "o/e/m", opened, errored, message, (!(opened || message)), (options.requireMessage && !message));
+                            onclose && onclose(ws, event);
+                            if (!errored) {
+                                if (!(opened || message)) {
+                                    ws.destroyCompletely();
+                                    this.config.LOG.error && log.warning(id, `WebSocket closed before opening, erroring: ${url}`);
+                                    retry();
+                                };
+                                if (options.requireMessage && !message) { //options.requireMessage && 
+                                    ws.destroyCompletely();
+                                    this.config.LOG.error && log.warning(id, `WebSocket closed before doing anything: ${url}`);
+                                    retry();
+                                };
+                            };
+                        } catch (error) {
+                            this.config.LOG.error && log.error(id, "error in onclose", error);
+                        };
+                    };
+
+                    ws.onmessage = (event) => {
+                        opened = true; message = true;
+                        try {
+                            onmessage && onmessage(ws, event);
+                        } catch (error) {
+                            this.config.LOG.error && log.error(id, "error in onmessage", error);
+                        };
+                    };
+                    return;
+                } catch (error) {
+                    if (attempts >= this.maxRetries) {
+                        reject(new Error(`Failed to connect to ${url} after ${this.maxRetries} attempts: ${error.message}`));
+                    };
+                };
+            };
+            retry();
+        });
+    }
+
+    createProxyRequest(url, options) {
+        return new ProxyRequest(this, url, options);
+    }
+}

package/src/providers/IPVanish.js

@@ -0,0 +1,36 @@
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import { IPVANISH_HOSTS } from '../constants.js'; // Moved big list to constants
+import scrambled from 'puppyscrambled'; // For Math.shuffleArray
+
+export default class IPVanish {
+    constructor(config) {
+        this.config = config;
+        this.hosts = [...IPVANISH_HOSTS];
+        this.hostUsage = {};
+    }
+
+    getHost(url) {
+        if (!this.hostUsage[url]) {
+            this.hostUsage[url] = scrambled.shuffleArray(JSON.parse(JSON.stringify(this.hosts)));
+        }
+        const takenHost = this.hostUsage[url].shift();
+        this.hostUsage[url].push(takenHost);
+        return takenHost;
+    }
+
+    getAgent(url) {
+        const conf = this.config.ipvanish;
+        if (!conf || !conf.use) return null;
+
+        if (conf.reducedmode && !(url.includes("googleapis") || url.includes("/game/"))) {
+            return null;
+        }
+
+        const creds = `${conf.creds.username}:${conf.creds.password}`;
+        const socksProxy = `socks5://${creds}@${this.getHost(url)}:1080`;
+        
+        const agent = new SocksProxyAgent(socksProxy);
+        agent.USINGSCRAPED = socksProxy;
+        return agent;
+    }
+}

package/src/providers/Scraper.js

@@ -0,0 +1,144 @@
+import fs from 'node:fs';
+import path from 'path';
+import { Worker } from 'worker_threads';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import log from 'puppylog';
+import id from 'puppyid';
+import { getLastSavedTimestamp } from 'puppymisc';
+import scrambled from 'puppyscrambled';
+
+export default class Scraper {
+    constructor(config, proxyDB) {
+        this.config = config;
+        this.proxyDB = proxyDB; // Reference to ProxyDatabase instance
+        this.hosts = [];
+        this.hostUsage = {};
+        
+        // Paths
+        this.storeDir = this.config.storeDir || './store';
+        this.outputPath = path.join(this.storeDir, 'scraped_proxies.json');
+    }
+
+    createAgent(proxyUrl) {
+        if (proxyUrl.startsWith('socks')) return new SocksProxyAgent(proxyUrl);
+        if (proxyUrl.startsWith('http:')) return new HttpProxyAgent(proxyUrl);
+        if (proxyUrl.startsWith('https:')) return new HttpsProxyAgent(proxyUrl);
+        return null;
+    }
+
+    genHostsScraper() {
+        try {
+            if (!fs.existsSync(this.outputPath)) return;
+            const data = fs.readFileSync(this.outputPath, 'utf8');
+            let newHosts = JSON.parse(data);
+
+            if (newHosts.length < 20) {
+                this.hosts = [...new Set(this.hosts.concat(newHosts))];
+                console.log("Scraper hosts merged!", this.hosts.length);
+            } else {
+                this.hosts = newHosts;
+                console.log("Scraper hosts updated!", this.hosts.length);
+            }
+            this.hostUsage = {};
+        } catch (error) {
+            console.error("Error loading scraper hosts:", error);
+            this.hosts = [];
+        }
+    }
+
+    // Alias for init
+    async loadScrapedProxies() {
+        this.genHostsScraper();
+    }
+
+    getHostAny(url = '') {
+        if (!this.hosts || this.hosts.length === 0) {
+            log.error(id, "No hosts available from scraper! IP would be exposed!");
+            // In a library, we shouldn't process.exit. We throw.
+            throw new Error("No hosts available from scraper");
+        }
+
+        if (!this.hostUsage[url]) {
+            this.hostUsage[url] = scrambled.shuffleArray(JSON.parse(JSON.stringify(this.hosts)));
+        }
+
+        const takenHost = this.hostUsage[url].shift();
+        this.hostUsage[url].push(takenHost);
+        return takenHost;
+    }
+
+    getHost(url = '', type = "socks") {
+        let max = 10000;
+        let count = 0;
+        while (count < max) {
+            const host = this.getHostAny(url);
+            if (host.startsWith(type)) {
+                return host;
+            }
+            count++;
+        }
+        log.error(id, `No matching host found that starts with '${type}'`);
+        if (type.startsWith("http")) {
+            return this.getHost(url, "socks");
+        }
+        return null;
+    }
+
+    getAgent(url = '', type = "socks") {
+        const conf = this.config.scraper;
+        if (conf && conf.use && this.hosts && this.hosts.length > 0) {
+            if (conf.reducedmode && !(url.includes("googleapis") || url.includes("/game/"))) {
+                return null;
+            }
+            const socksProxy = this.getHost(url, type);
+            if (!socksProxy) return null;
+
+            const agent = this.createAgent(socksProxy);
+            agent.USINGSCRAPED = socksProxy;
+            return agent;
+        }
+        return null;
+    }
+
+    // Worker Orchestration
+    async runCollector(force = false) {
+        let lastSaved = (Date.now() - getLastSavedTimestamp(this.outputPath)) / 1000;
+        if (lastSaved && !force) {
+            console.log("lastSaved", lastSaved);
+            let timeBetweenScrapes = this.config.scraper.timeBetweenScrapes * 60; // 25 mins
+            if (lastSaved < timeBetweenScrapes) {
+                console.log(`Proxies saved < ${timeBetweenScrapes}s ago, skipping.`);
+                return;
+            }
+        }
+
+        return new Promise((resolve, reject) => {
+            try {
+                // Pointing to the worker file. 
+                // Note: In a real npm package, ensure 'src/workers/collector.js' is included
+                const workerUrl = new URL('../workers/collector.js', import.meta.url);
+                const worker = new Worker(workerUrl);
+                
+                worker.postMessage(["deploy", { config: this.config }]);
+
+                worker.on('message', async (message) => {
+                    worker.terminate();
+                    // The worker returns raw proxies, we process them into DB
+                    await this.proxyDB.dealWithProxies(message, this.outputPath);
+                    this.genHostsScraper(); // Reload memory
+                    resolve();
+                });
+
+                worker.on('error', (error) => {
+                    console.error("Worker error:", error);
+                    reject(error);
+                });
+            } catch (error) {
+                console.error("Proxy collector launch error:", error);
+                reject(error);
+            }
+        });
+    }
+}

package/src/utils.js

@@ -0,0 +1,12 @@
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+
+export function createBasicAgent(proxyUrl) {
+    if (!proxyUrl) return null;
+    
+    if (proxyUrl.startsWith("socks")) return new SocksProxyAgent(proxyUrl);
+    if (proxyUrl.startsWith("https")) return new HttpsProxyAgent(proxyUrl);
+    if (proxyUrl.startsWith("http")) return new HttpProxyAgent(proxyUrl);
+    return null;
+}