webpeel 0.21.8 → 0.21.10

package/dist/core/browser-fetch.js

@@ -5,7 +5,7 @@
 import { TimeoutError, BlockedError, NetworkError, WebPeelError } from '../types.js';
 import { detectChallenge } from './challenge-detection.js';
 import { getRealisticUserAgent } from './user-agents.js';
-import { getRandomUserAgent, applyStealthScripts, takePooledPage, ensurePagePool, recyclePooledPage, getBrowser, getStealthBrowser, getProfileBrowser, PAGE_POOL_SIZE, MAX_CONCURRENT_PAGES, getPooledPagesCount, } from './browser-pool.js';
+import { getRandomUserAgent, applyStealthScripts, takePooledPage, ensurePagePool, recyclePooledPage, getBrowser, getStealthBrowser, getStealthPlaywright, getProfileBrowser, PAGE_POOL_SIZE, MAX_CONCURRENT_PAGES, getPooledPagesCount, ANTI_DETECTION_ARGS, getRandomViewport, } from './browser-pool.js';
 // Proprietary stealth module — gitignored, loaded conditionally
 let applyStealthPatches;
 let applyAcceptLanguageHeader;
@@ -90,6 +90,8 @@ export async function browserFetch(url, options = {}) {
     const usingProfileBrowser = !!profileDir;
     // Owned context created when storageState injection is requested
     let ownedContext;
+    // Owned browser launched when proxy is specified (dedicated browser with proxy at launch level)
+    let ownedBrowser;
     try {
         const browser = usingProfileBrowser
             ? await getProfileBrowser(profileDir, headed, stealth)
@@ -135,11 +137,22 @@ export async function browserFetch(url, options = {}) {
                     log.debug('proxy URL parse failed, using as-is:', e instanceof Error ? e.message : e);
                     playwrightProxy = { server: proxy };
                 }
-                // Create an isolated context with the proxy and optional storageState
-                ownedContext = await browser.newContext({
-                    ...pageOptions,
+                // Launch a DEDICATED fresh browser with proxy at the launch level.
+                // Context-level proxy is unreliable for anti-bot sites — they check the browser's
+                // IP at connection time (set at launch), not at context creation.
+                const pw = stealth ? await getStealthPlaywright() : (await import('playwright')).chromium;
+                const vp = getRandomViewport();
+                ownedBrowser = await pw.launch({
+                    headless: true,
+                    args: [...ANTI_DETECTION_ARGS, `--window-size=${vp.width},${vp.height}`],
                     proxy: playwrightProxy,
-                    viewport: { width: effectiveViewportWidth, height: effectiveViewportHeight },
+                });
+                ownedContext = await ownedBrowser.newContext({
+                    userAgent: validatedUserAgent || getRandomUserAgent(),
+                    locale: 'en-US',
+                    timezoneId: 'America/New_York',
+                    javaScriptEnabled: true,
+                    viewport: { width: effectiveViewportWidth || vp.width, height: effectiveViewportHeight || vp.height },
                     ...(storageState ? { storageState } : {}),
                 });
                 page = await ownedContext.newPage();
@@ -340,6 +353,29 @@ export async function browserFetch(url, options = {}) {
                 await page.waitForTimeout(extraDelayMs);
                 throwIfAborted();
             }
+            // Human-like delay for proxied requests (helps bypass bot detection on strict sites)
+            if (proxy) {
+                // Realistic human behavior to bypass behavioral analysis
+                const humanDelay = 800 + Math.random() * 1200;
+                await page.waitForTimeout(humanDelay);
+                throwIfAborted();
+                // Realistic mouse movement (simulate human cursor)
+                try {
+                    const vw = await page.evaluate(() => window.innerWidth);
+                    const vh = await page.evaluate(() => window.innerHeight);
+                    await page.mouse.move(100 + Math.random() * (vw - 200), 100 + Math.random() * (vh - 200), { steps: 5 + Math.floor(Math.random() * 10) });
+                    // Small scroll to trigger lazy-loaded content
+                    await page.evaluate(() => window.scrollBy(0, 200 + Math.random() * 400));
+                    await page.waitForTimeout(300 + Math.random() * 500);
+                    throwIfAborted();
+                    // Second mouse move
+                    await page.mouse.move(50 + Math.random() * (vw - 100), 50 + Math.random() * (vh - 100), { steps: 3 + Math.floor(Math.random() * 5) });
+                }
+                catch {
+                    // Non-fatal: mouse/scroll simulation failed
+                }
+                throwIfAborted();
+            }
             // Wait for additional time if requested (for dynamic content / screenshots)
             if (waitMs > 0) {
                 await page.waitForTimeout(waitMs);
@@ -447,7 +483,8 @@ export async function browserFetch(url, options = {}) {
                 contentType: fetchContentType,
                 screenshot: screenshotBuffer,
                 page,
-                browser,
+                // Use ownedBrowser for proxy case, otherwise the shared browser
+                browser: ownedBrowser ?? browser,
                 ...(fetchAutoInteract !== undefined ? { autoInteract: fetchAutoInteract } : {}),
             };
         }
@@ -492,6 +529,10 @@ export async function browserFetch(url, options = {}) {
                 await page.close().catch(() => { });
             }
         }
+        // Close the dedicated proxy browser if one was launched (not when keeping page open)
+        if (ownedBrowser && !keepPageOpen) {
+            await ownedBrowser.close().catch(() => { });
+        }
         activePagesCount--;
     }
 }

package/dist/core/browser-pool.d.ts

@@ -3,10 +3,12 @@
  * Handles Playwright loading, browser instances, and the idle page pool.
  */
 import type { Browser, Page } from 'playwright';
+type ChromiumType = typeof import('playwright').chromium;
 import { closePool } from './http-fetch.js';
 export { closePool };
 /** Whether Playwright has been loaded (for diagnostics). */
 export declare let playwrightLoaded: boolean;
+export declare function getStealthPlaywright(): Promise<ChromiumType>;
 /**
  * Returns a realistic Chrome user agent.
  * Delegates to the curated user-agents module so stealth mode never exposes

package/dist/core/browser-pool.js

@@ -20,7 +20,7 @@ async function getPlaywright() {
     }
     return _chromium;
 }
-async function getStealthPlaywright() {
+export async function getStealthPlaywright() {
     if (!_stealthChromium) {
         const pwExtra = await import('playwright-extra');
         const StealthPlugin = (await import('puppeteer-extra-plugin-stealth')).default;
@@ -54,7 +54,7 @@ export const ANTI_DETECTION_ARGS = [
     '--disable-gpu',
     '--start-maximized',
     // Chrome branding / stealth hardening
-    '--disable-features=ChromeUserAgentDataBranding',
+    '--disable-features=ChromeUserAgentDataBranding,IsolateOrigins,site-per-process',
     '--disable-component-extensions-with-background-pages',
     '--disable-default-apps',
     '--disable-extensions',
@@ -143,6 +143,95 @@ export async function applyStealthScripts(page) {
       });
     })();
   `);
+    // 3. Hide navigator.webdriver (THE #1 BOT SIGNAL)
+    await page.addInitScript(`
+    Object.defineProperty(navigator, 'webdriver', {
+      get: () => false,
+      configurable: true,
+    });
+    try { delete Object.getPrototypeOf(navigator).webdriver; } catch (e) {}
+  `);
+    // 4. Fake navigator.plugins (empty = bot signal, real Chrome has plugins)
+    await page.addInitScript(`
+    Object.defineProperty(navigator, 'plugins', {
+      get: () => {
+        var arr = [
+          { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
+          { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
+          { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
+        ];
+        arr.item = function(i) { return arr[i] || null; };
+        arr.namedItem = function(n) { return arr.find(function(p) { return p.name === n; }) || null; };
+        arr.refresh = function() {};
+        return arr;
+      },
+      configurable: true,
+    });
+  `);
+    // 5. Fake navigator.languages
+    await page.addInitScript(`
+    Object.defineProperty(navigator, 'languages', {
+      get: () => ['en-US', 'en'],
+      configurable: true,
+    });
+  `);
+    // 6. Fake window.chrome object (missing in headless = detected)
+    await page.addInitScript(`
+    if (!window.chrome) {
+      window.chrome = {
+        app: {
+          isInstalled: false,
+          InstallState: { INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
+          RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
+        },
+        runtime: {
+          OnInstalledReason: {}, OnRestartRequiredReason: {}, PlatformArch: {},
+          PlatformNaclArch: {}, PlatformOs: {}, RequestUpdateCheckStatus: {},
+          connect: function() {}, sendMessage: function() {}
+        },
+      };
+    }
+  `);
+    // 7. Fix permissions query (notifications should be 'prompt' not 'denied')
+    await page.addInitScript(`
+    try {
+      var originalQuery = window.Permissions && window.Permissions.prototype && window.Permissions.prototype.query;
+      if (originalQuery) {
+        window.Permissions.prototype.query = function(params) {
+          if (params && params.name === 'notifications') {
+            return Promise.resolve({ state: Notification.permission });
+          }
+          return originalQuery.call(this, params);
+        };
+      }
+    } catch (e) {}
+  `);
+    // 8. WebGL vendor/renderer spoofing (headless shows "Google SwiftShader")
+    await page.addInitScript(`
+    try {
+      var getParameter = WebGLRenderingContext.prototype.getParameter;
+      WebGLRenderingContext.prototype.getParameter = function(parameter) {
+        if (parameter === 37445) return 'Intel Inc.';
+        if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+        return getParameter.call(this, parameter);
+      };
+      if (typeof WebGL2RenderingContext !== 'undefined') {
+        var getParameter2 = WebGL2RenderingContext.prototype.getParameter;
+        WebGL2RenderingContext.prototype.getParameter = function(parameter) {
+          if (parameter === 37445) return 'Intel Inc.';
+          if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+          return getParameter2.call(this, parameter);
+        };
+      }
+    } catch (e) {}
+  `);
+    // 9. Hide automation-related properties
+    await page.addInitScript(`
+    try { Object.defineProperty(document, '$cdc_asdjflasutopfhvcZLmcfl_', { get: () => undefined }); } catch (e) {}
+    try { delete window.callPhantom; } catch (e) {}
+    try { delete window._phantom; } catch (e) {}
+    try { delete window.__nightmare; } catch (e) {}
+  `);
 }
 // ── Page pool constants & state ───────────────────────────────────────────────
 export const MAX_CONCURRENT_PAGES = 5;

package/dist/core/domain-extractors.js

@@ -308,6 +308,7 @@ async function twitterExtractor(html, url) {
             if (fxData && fxData.code === 200 && fxData.user) {
                 const u = fxData.user;
                 const structured = {
+                    title: `${u.name || ''} (@${u.screen_name || ''}) on X/Twitter`,
                     name: u.name || '',
                     handle: '@' + (u.screen_name || ''),
                     bio: u.description || '',
@@ -1500,6 +1501,7 @@ async function npmExtractor(_html, url) {
         }
         catch { /* optional */ }
         const structured = {
+            title: `${data.name}@${latest || 'unknown'}`,
             name: data.name,
             description: data.description || '',
             version: latest || 'unknown',

package/dist/core/strategies.js

@@ -76,6 +76,7 @@ function shouldForceBrowser(url) {
             'chewy.com', // Amazon subsidiary
             'aliexpress.com', // anti-bot
             'wish.com', // anti-bot
+            'cargurus.com', // aggressive bot detection
         ];
         for (const domain of stealthDomains) {
             if (hostname === domain || hostname.endsWith(`.${domain}`)) {

package/dist/core/structured-extract.js

@@ -296,7 +296,13 @@ function heuristicExtractNumber(fieldName, content) {
     }
     // Year
     if (/year/.test(lf)) {
-        const m = content.match(/\b(20\d{2})\b/);
+        // Match 4-digit years (1900-2099), prefer explicit "Year: YYYY" pattern first
+        const explicit = content.match(/\bYear[:\s]+(\d{4})\b/i);
+        if (explicit?.[1]) {
+            const n = parseInt(explicit[1]);
+            return isNaN(n) ? null : n;
+        }
+        const m = content.match(/\b((?:19|20)\d{2})\b/);
         if (m?.[1]) {
             const n = parseInt(m[1]);
             return isNaN(n) ? null : n;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.8",
+  "version": "0.21.10",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",