webpeel 0.21.7 → 0.21.9

package/dist/core/browser-fetch.js

@@ -5,7 +5,7 @@
 import { TimeoutError, BlockedError, NetworkError, WebPeelError } from '../types.js';
 import { detectChallenge } from './challenge-detection.js';
 import { getRealisticUserAgent } from './user-agents.js';
-import { getRandomUserAgent, applyStealthScripts, takePooledPage, ensurePagePool, recyclePooledPage, getBrowser, getStealthBrowser, getProfileBrowser, PAGE_POOL_SIZE, MAX_CONCURRENT_PAGES, getPooledPagesCount, } from './browser-pool.js';
+import { getRandomUserAgent, applyStealthScripts, takePooledPage, ensurePagePool, recyclePooledPage, getBrowser, getStealthBrowser, getStealthPlaywright, getProfileBrowser, PAGE_POOL_SIZE, MAX_CONCURRENT_PAGES, getPooledPagesCount, ANTI_DETECTION_ARGS, getRandomViewport, } from './browser-pool.js';
 // Proprietary stealth module — gitignored, loaded conditionally
 let applyStealthPatches;
 let applyAcceptLanguageHeader;
@@ -90,6 +90,8 @@ export async function browserFetch(url, options = {}) {
     const usingProfileBrowser = !!profileDir;
     // Owned context created when storageState injection is requested
     let ownedContext;
+    // Owned browser launched when proxy is specified (dedicated browser with proxy at launch level)
+    let ownedBrowser;
     try {
         const browser = usingProfileBrowser
             ? await getProfileBrowser(profileDir, headed, stealth)
@@ -135,11 +137,22 @@ export async function browserFetch(url, options = {}) {
                     log.debug('proxy URL parse failed, using as-is:', e instanceof Error ? e.message : e);
                     playwrightProxy = { server: proxy };
                 }
-                // Create an isolated context with the proxy and optional storageState
-                ownedContext = await browser.newContext({
-                    ...pageOptions,
+                // Launch a DEDICATED fresh browser with proxy at the launch level.
+                // Context-level proxy is unreliable for anti-bot sites — they check the browser's
+                // IP at connection time (set at launch), not at context creation.
+                const pw = stealth ? await getStealthPlaywright() : (await import('playwright')).chromium;
+                const vp = getRandomViewport();
+                ownedBrowser = await pw.launch({
+                    headless: true,
+                    args: [...ANTI_DETECTION_ARGS, `--window-size=${vp.width},${vp.height}`],
                     proxy: playwrightProxy,
-                    viewport: { width: effectiveViewportWidth, height: effectiveViewportHeight },
+                });
+                ownedContext = await ownedBrowser.newContext({
+                    userAgent: validatedUserAgent || getRandomUserAgent(),
+                    locale: 'en-US',
+                    timezoneId: 'America/New_York',
+                    javaScriptEnabled: true,
+                    viewport: { width: effectiveViewportWidth || vp.width, height: effectiveViewportHeight || vp.height },
                     ...(storageState ? { storageState } : {}),
                 });
                 page = await ownedContext.newPage();
@@ -340,6 +353,29 @@ export async function browserFetch(url, options = {}) {
                 await page.waitForTimeout(extraDelayMs);
                 throwIfAborted();
             }
+            // Human-like delay for proxied requests (helps bypass bot detection on strict sites)
+            if (proxy) {
+                // Realistic human behavior to bypass behavioral analysis
+                const humanDelay = 800 + Math.random() * 1200;
+                await page.waitForTimeout(humanDelay);
+                throwIfAborted();
+                // Realistic mouse movement (simulate human cursor)
+                try {
+                    const vw = await page.evaluate(() => window.innerWidth);
+                    const vh = await page.evaluate(() => window.innerHeight);
+                    await page.mouse.move(100 + Math.random() * (vw - 200), 100 + Math.random() * (vh - 200), { steps: 5 + Math.floor(Math.random() * 10) });
+                    // Small scroll to trigger lazy-loaded content
+                    await page.evaluate(() => window.scrollBy(0, 200 + Math.random() * 400));
+                    await page.waitForTimeout(300 + Math.random() * 500);
+                    throwIfAborted();
+                    // Second mouse move
+                    await page.mouse.move(50 + Math.random() * (vw - 100), 50 + Math.random() * (vh - 100), { steps: 3 + Math.floor(Math.random() * 5) });
+                }
+                catch {
+                    // Non-fatal: mouse/scroll simulation failed
+                }
+                throwIfAborted();
+            }
             // Wait for additional time if requested (for dynamic content / screenshots)
             if (waitMs > 0) {
                 await page.waitForTimeout(waitMs);
@@ -447,7 +483,8 @@ export async function browserFetch(url, options = {}) {
                 contentType: fetchContentType,
                 screenshot: screenshotBuffer,
                 page,
-                browser,
+                // Use ownedBrowser for proxy case, otherwise the shared browser
+                browser: ownedBrowser ?? browser,
                 ...(fetchAutoInteract !== undefined ? { autoInteract: fetchAutoInteract } : {}),
             };
         }
@@ -492,6 +529,10 @@ export async function browserFetch(url, options = {}) {
                 await page.close().catch(() => { });
             }
         }
+        // Close the dedicated proxy browser if one was launched (not when keeping page open)
+        if (ownedBrowser && !keepPageOpen) {
+            await ownedBrowser.close().catch(() => { });
+        }
         activePagesCount--;
     }
 }

package/dist/core/browser-pool.d.ts

@@ -3,10 +3,12 @@
  * Handles Playwright loading, browser instances, and the idle page pool.
  */
 import type { Browser, Page } from 'playwright';
+type ChromiumType = typeof import('playwright').chromium;
 import { closePool } from './http-fetch.js';
 export { closePool };
 /** Whether Playwright has been loaded (for diagnostics). */
 export declare let playwrightLoaded: boolean;
+export declare function getStealthPlaywright(): Promise<ChromiumType>;
 /**
  * Returns a realistic Chrome user agent.
  * Delegates to the curated user-agents module so stealth mode never exposes

package/dist/core/browser-pool.js

@@ -20,7 +20,7 @@ async function getPlaywright() {
     }
     return _chromium;
 }
-async function getStealthPlaywright() {
+export async function getStealthPlaywright() {
     if (!_stealthChromium) {
         const pwExtra = await import('playwright-extra');
         const StealthPlugin = (await import('puppeteer-extra-plugin-stealth')).default;
@@ -54,7 +54,7 @@ export const ANTI_DETECTION_ARGS = [
     '--disable-gpu',
     '--start-maximized',
     // Chrome branding / stealth hardening
-    '--disable-features=ChromeUserAgentDataBranding',
+    '--disable-features=ChromeUserAgentDataBranding,IsolateOrigins,site-per-process',
     '--disable-component-extensions-with-background-pages',
     '--disable-default-apps',
     '--disable-extensions',
@@ -143,6 +143,95 @@ export async function applyStealthScripts(page) {
       });
     })();
   `);
+    // 3. Hide navigator.webdriver (THE #1 BOT SIGNAL)
+    await page.addInitScript(`
+    Object.defineProperty(navigator, 'webdriver', {
+      get: () => false,
+      configurable: true,
+    });
+    try { delete Object.getPrototypeOf(navigator).webdriver; } catch (e) {}
+  `);
+    // 4. Fake navigator.plugins (empty = bot signal, real Chrome has plugins)
+    await page.addInitScript(`
+    Object.defineProperty(navigator, 'plugins', {
+      get: () => {
+        var arr = [
+          { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
+          { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
+          { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
+        ];
+        arr.item = function(i) { return arr[i] || null; };
+        arr.namedItem = function(n) { return arr.find(function(p) { return p.name === n; }) || null; };
+        arr.refresh = function() {};
+        return arr;
+      },
+      configurable: true,
+    });
+  `);
+    // 5. Fake navigator.languages
+    await page.addInitScript(`
+    Object.defineProperty(navigator, 'languages', {
+      get: () => ['en-US', 'en'],
+      configurable: true,
+    });
+  `);
+    // 6. Fake window.chrome object (missing in headless = detected)
+    await page.addInitScript(`
+    if (!window.chrome) {
+      window.chrome = {
+        app: {
+          isInstalled: false,
+          InstallState: { INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
+          RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
+        },
+        runtime: {
+          OnInstalledReason: {}, OnRestartRequiredReason: {}, PlatformArch: {},
+          PlatformNaclArch: {}, PlatformOs: {}, RequestUpdateCheckStatus: {},
+          connect: function() {}, sendMessage: function() {}
+        },
+      };
+    }
+  `);
+    // 7. Fix permissions query (notifications should be 'prompt' not 'denied')
+    await page.addInitScript(`
+    try {
+      var originalQuery = window.Permissions && window.Permissions.prototype && window.Permissions.prototype.query;
+      if (originalQuery) {
+        window.Permissions.prototype.query = function(params) {
+          if (params && params.name === 'notifications') {
+            return Promise.resolve({ state: Notification.permission });
+          }
+          return originalQuery.call(this, params);
+        };
+      }
+    } catch (e) {}
+  `);
+    // 8. WebGL vendor/renderer spoofing (headless shows "Google SwiftShader")
+    await page.addInitScript(`
+    try {
+      var getParameter = WebGLRenderingContext.prototype.getParameter;
+      WebGLRenderingContext.prototype.getParameter = function(parameter) {
+        if (parameter === 37445) return 'Intel Inc.';
+        if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+        return getParameter.call(this, parameter);
+      };
+      if (typeof WebGL2RenderingContext !== 'undefined') {
+        var getParameter2 = WebGL2RenderingContext.prototype.getParameter;
+        WebGL2RenderingContext.prototype.getParameter = function(parameter) {
+          if (parameter === 37445) return 'Intel Inc.';
+          if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+          return getParameter2.call(this, parameter);
+        };
+      }
+    } catch (e) {}
+  `);
+    // 9. Hide automation-related properties
+    await page.addInitScript(`
+    try { Object.defineProperty(document, '$cdc_asdjflasutopfhvcZLmcfl_', { get: () => undefined }); } catch (e) {}
+    try { delete window.callPhantom; } catch (e) {}
+    try { delete window._phantom; } catch (e) {}
+    try { delete window.__nightmare; } catch (e) {}
+  `);
 }
 // ── Page pool constants & state ───────────────────────────────────────────────
 export const MAX_CONCURRENT_PAGES = 5;

package/dist/core/proxy-config.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Shared Webshare residential proxy configuration.
+ *
+ * WebPeel uses Webshare residential proxies (configured via env vars) to route
+ * requests through US residential IPs, bypassing datacenter IP blocks from
+ * DuckDuckGo, Amazon, BestBuy, CarGurus, and other sites with anti-bot detection.
+ *
+ * Proxy credentials are loaded from environment variables:
+ *   WEBSHARE_PROXY_HOST  — proxy hostname (e.g. p.webshare.io)
+ *   WEBSHARE_PROXY_PORT  — base port number (e.g. 10000)
+ *   WEBSHARE_PROXY_USER  — proxy username (without slot suffix)
+ *   WEBSHARE_PROXY_PASS  — proxy password
+ *   WEBSHARE_PROXY_SLOTS — number of available US residential slots
+ *
+ * With the Webshare backbone plan each US slot has its own port:
+ *   slot N → port (WEBSHARE_PROXY_PORT + N - 1), username: USER-US-N
+ */
+export interface ProxyConfig {
+    /** Proxy server URL in the format "http://host:port" */
+    server: string;
+    /** Proxy username (includes slot suffix, e.g. "user-US-42") */
+    username: string;
+    /** Proxy password */
+    password: string;
+}
+/**
+ * Get a random Webshare residential proxy config.
+ * Returns null if the proxy is not configured (env vars missing or slots = 0).
+ *
+ * Uses random slot selection across all available US slots for even load
+ * distribution — same approach as youtube.ts proxyRequestSlotted().
+ */
+export declare function getWebshareProxy(): ProxyConfig | null;
+/**
+ * Check if Webshare proxies are configured (env vars are present and non-empty).
+ * Does NOT guarantee the proxy is reachable — just that credentials are set.
+ */
+export declare function hasWebshareProxy(): boolean;
+/**
+ * Convert a ProxyConfig to a Playwright-compatible proxy object.
+ * Useful for passing directly to browser.newContext({ proxy: ... }).
+ */
+export declare function toPlaywrightProxy(config: ProxyConfig): {
+    server: string;
+    username: string;
+    password: string;
+};
+/**
+ * Get a random Webshare proxy as a fully-qualified URL string with embedded
+ * credentials. The format is: `http://username:password@host:port`
+ *
+ * Useful for passing to strategies.ts proxy option (which expects a URL string).
+ * Returns null if proxies are not configured.
+ */
+export declare function getWebshareProxyUrl(): string | null;

package/dist/core/proxy-config.js

@@ -0,0 +1,79 @@
+/**
+ * Shared Webshare residential proxy configuration.
+ *
+ * WebPeel uses Webshare residential proxies (configured via env vars) to route
+ * requests through US residential IPs, bypassing datacenter IP blocks from
+ * DuckDuckGo, Amazon, BestBuy, CarGurus, and other sites with anti-bot detection.
+ *
+ * Proxy credentials are loaded from environment variables:
+ *   WEBSHARE_PROXY_HOST  — proxy hostname (e.g. p.webshare.io)
+ *   WEBSHARE_PROXY_PORT  — base port number (e.g. 10000)
+ *   WEBSHARE_PROXY_USER  — proxy username (without slot suffix)
+ *   WEBSHARE_PROXY_PASS  — proxy password
+ *   WEBSHARE_PROXY_SLOTS — number of available US residential slots
+ *
+ * With the Webshare backbone plan each US slot has its own port:
+ *   slot N → port (WEBSHARE_PROXY_PORT + N - 1), username: USER-US-N
+ */
+/**
+ * Get a random Webshare residential proxy config.
+ * Returns null if the proxy is not configured (env vars missing or slots = 0).
+ *
+ * Uses random slot selection across all available US slots for even load
+ * distribution — same approach as youtube.ts proxyRequestSlotted().
+ */
+export function getWebshareProxy() {
+    const host = process.env.WEBSHARE_PROXY_HOST;
+    const user = process.env.WEBSHARE_PROXY_USER;
+    const pass = process.env.WEBSHARE_PROXY_PASS;
+    const basePort = parseInt(process.env.WEBSHARE_PROXY_PORT || '10000', 10);
+    const slots = parseInt(process.env.WEBSHARE_PROXY_SLOTS || '0', 10);
+    if (!host || !user || !pass || slots <= 0)
+        return null;
+    const slot = Math.floor(Math.random() * slots) + 1;
+    const port = basePort + slot - 1;
+    return {
+        server: `http://${host}:${port}`,
+        username: `${user}-US-${slot}`,
+        password: pass,
+    };
+}
+/**
+ * Check if Webshare proxies are configured (env vars are present and non-empty).
+ * Does NOT guarantee the proxy is reachable — just that credentials are set.
+ */
+export function hasWebshareProxy() {
+    return !!(process.env.WEBSHARE_PROXY_HOST &&
+        process.env.WEBSHARE_PROXY_USER &&
+        process.env.WEBSHARE_PROXY_PASS);
+}
+/**
+ * Convert a ProxyConfig to a Playwright-compatible proxy object.
+ * Useful for passing directly to browser.newContext({ proxy: ... }).
+ */
+export function toPlaywrightProxy(config) {
+    return {
+        server: config.server,
+        username: config.username,
+        password: config.password,
+    };
+}
+/**
+ * Get a random Webshare proxy as a fully-qualified URL string with embedded
+ * credentials. The format is: `http://username:password@host:port`
+ *
+ * Useful for passing to strategies.ts proxy option (which expects a URL string).
+ * Returns null if proxies are not configured.
+ */
+export function getWebshareProxyUrl() {
+    const config = getWebshareProxy();
+    if (!config)
+        return null;
+    try {
+        const url = new URL(config.server);
+        return `http://${encodeURIComponent(config.username)}:${encodeURIComponent(config.password)}@${url.host}`;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/core/search-provider.js

@@ -15,6 +15,7 @@
 import { fetch as undiciFetch } from 'undici';
 import { load } from 'cheerio';
 import { getStealthBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
+import { getWebshareProxy } from './proxy-config.js';
 import { createLogger } from './logger.js';
 const log = createLogger('search');
 function decodeHtmlEntities(input) {
@@ -236,10 +237,12 @@ export class StealthSearchProvider {
             const browser = await getStealthBrowser();
             const params = new URLSearchParams({ q: query });
             const url = `https://html.duckduckgo.com/html/?${params.toString()}`;
+            const proxy = getWebshareProxy();
             ctx = await browser.newContext({
                 userAgent: getRandomUserAgent(),
                 locale: 'en-US',
                 timezoneId: 'America/New_York',
+                ...(proxy ? { proxy: { server: proxy.server, username: proxy.username, password: proxy.password } } : {}),
             });
             const page = await ctx.newPage();
             await applyStealthScripts(page);
@@ -303,10 +306,12 @@ export class StealthSearchProvider {
             const browser = await getStealthBrowser();
             const params = new URLSearchParams({ q: query });
             const url = `https://www.bing.com/search?${params.toString()}`;
+            const proxy = getWebshareProxy();
             ctx = await browser.newContext({
                 userAgent: getRandomUserAgent(),
                 locale: 'en-US',
                 timezoneId: 'America/New_York',
+                ...(proxy ? { proxy: { server: proxy.server, username: proxy.username, password: proxy.password } } : {}),
             });
             const page = await ctx.newPage();
             await applyStealthScripts(page);
@@ -380,10 +385,12 @@ export class StealthSearchProvider {
             const browser = await getStealthBrowser();
             const params = new URLSearchParams({ q: query });
             const url = `https://www.ecosia.org/search?${params.toString()}`;
+            const proxy = getWebshareProxy();
             ctx = await browser.newContext({
                 userAgent: getRandomUserAgent(),
                 locale: 'en-US',
                 timezoneId: 'America/New_York',
+                ...(proxy ? { proxy: { server: proxy.server, username: proxy.username, password: proxy.password } } : {}),
             });
             const page = await ctx.newPage();
             await applyStealthScripts(page);

package/dist/core/strategies.js

@@ -10,6 +10,7 @@ import { simpleFetch, browserFetch, retryFetch } from './fetcher.js';
 import { getCached, setCached as setBasicCache } from './cache.js';
 import { resolveAndCache } from './dns-cache.js';
 import { BlockedError, NetworkError } from '../types.js';
+import { getWebshareProxyUrl } from './proxy-config.js';
 import { detectChallenge } from './challenge-detection.js';
 import { getStrategyHooks, } from './strategy-hooks.js';
 import { createLogger } from './logger.js';
@@ -75,6 +76,7 @@ function shouldForceBrowser(url) {
             'chewy.com', // Amazon subsidiary
             'aliexpress.com', // anti-bot
             'wish.com', // anti-bot
+            'cargurus.com', // aggressive bot detection
         ];
         for (const domain of stealthDomains) {
             if (hostname === domain || hostname.endsWith(`.${domain}`)) {
@@ -310,10 +312,15 @@ async function fetchWithBrowserStrategy(url, options) {
 export async function smartFetch(url, options = {}) {
     const { forceBrowser = false, stealth = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, actions, keepPageOpen = false, noCache = false, raceTimeoutMs = 2000, profileDir, headed = false, storageState, proxy, proxies, device, viewportWidth, viewportHeight, waitUntil, waitSelector, blockResources, cloaked = false, cycle = false, tls = false, noEscalate = false, } = options;
     const usePeelTLS = tls || cycle;
-    // Build effective proxy list: explicit proxies array, or single proxy, or empty
+    // Build effective proxy list: explicit proxies array, or single proxy, or empty.
+    // When no explicit proxy is configured and Webshare is available, automatically
+    // add it as a fallback: try direct connection first (fast), then Webshare on block.
     const effectiveProxies = proxies?.length ? proxies :
         proxy ? [proxy] :
-            [undefined]; // undefined = direct connection (no proxy)
+            (() => {
+                const wsUrl = getWebshareProxyUrl();
+                return wsUrl ? [undefined, wsUrl] : [undefined];
+            })();
     const firstProxy = effectiveProxies[0];
     const hooks = getStrategyHooks();
     const fetchStartMs = Date.now();

package/dist/core/youtube.js

@@ -15,6 +15,7 @@ import { join } from 'node:path';
 import { fetchTranscript as ytpFetchTranscript } from 'youtube-transcript-plus';
 import { simpleFetch } from './fetcher.js';
 import { getBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
+import { hasWebshareProxy as _hasWebshareProxy } from './proxy-config.js';
 import { createLogger } from './logger.js';
 // ---------------------------------------------------------------------------
 // yt-dlp startup diagnostics
@@ -239,8 +240,10 @@ export function extractSummary(fullText) {
 // ---------------------------------------------------------------------------
 // Proxy-based InnerTube transcript extraction
 // ---------------------------------------------------------------------------
-// Webshare residential proxy config — reads from env vars on Render.
+// Webshare residential proxy config — reads from env vars via proxy-config.ts.
 // Locally, falls back to direct fetch (residential IP already works).
+// These constants are kept for use in proxyRequestSlotted() which does
+// low-level HTTP CONNECT tunneling (not Playwright-level proxy).
 const PROXY_HOST = process.env.WEBSHARE_PROXY_HOST || 'p.webshare.io';
 const PROXY_BASE_PORT = parseInt(process.env.WEBSHARE_PROXY_PORT || '10000', 10);
 const PROXY_USER = process.env.WEBSHARE_PROXY_USER || '';
@@ -249,7 +252,8 @@ const PROXY_PASS = process.env.WEBSHARE_PROXY_PASS || '';
 // slot N → port (PROXY_BASE_PORT + N - 1), username: USER-US-N
 const PROXY_MAX_US_SLOTS = parseInt(process.env.WEBSHARE_PROXY_SLOTS || '44744', 10);
 function isProxyConfigured() {
-    return !!(PROXY_USER && PROXY_PASS);
+    // Delegate to the shared proxy-config helper for consistency
+    return _hasWebshareProxy();
 }
 /**
  * Make an HTTP(S) request through the Webshare CONNECT proxy with a specific

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.7",
+  "version": "0.21.9",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",