webpeel 0.7.0 → 0.7.1

package/dist/core/fetcher.js

@@ -1,11 +1,19 @@
 /**
  * Core fetching logic: simple HTTP and browser-based fetching
  */
+// Force IPv4-first DNS resolution globally.
+// Prevents IPv6 connection failures (TLS errors, timeouts) on hosts that
+// advertise AAAA records but can't actually route IPv6 (e.g. Render containers).
+// Must run before any network library is used.
+import dns from 'dns';
+dns.setDefaultResultOrder('ipv4first');
 import { chromium } from 'playwright';
 import { chromium as stealthChromium } from 'playwright-extra';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
-import { fetch as undiciFetch } from 'undici';
+import { fetch as undiciFetch, Agent } from 'undici';
 import { TimeoutError, BlockedError, NetworkError, WebPeelError } from '../types.js';
+import { getCached } from './cache.js';
+import { cachedLookup, resolveAndCache, startDnsWarmup } from './dns-cache.js';
 // Add stealth plugin to playwright-extra
 stealthChromium.use(StealthPlugin());
 const USER_AGENTS = [
@@ -18,6 +26,102 @@ const USER_AGENTS = [
 function getRandomUserAgent() {
     return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
 }
+function createHttpPool() {
+    return new Agent({
+        connections: 20,
+        pipelining: 6,
+        keepAliveTimeout: 60000,
+        keepAliveMaxTimeout: 60000,
+        allowH2: true,
+        connect: {
+            lookup: cachedLookup,
+        },
+    });
+}
+let httpPool = createHttpPool();
+startDnsWarmup();
+const CONDITIONAL_CACHE_MAX_ENTRIES = 2000;
+const conditionalValidatorsByUrl = new Map();
+function normalizeUrlForConditionalCache(url) {
+    try {
+        const normalized = new URL(url);
+        normalized.hash = '';
+        normalized.hostname = normalized.hostname.toLowerCase();
+        if ((normalized.protocol === 'http:' && normalized.port === '80') ||
+            (normalized.protocol === 'https:' && normalized.port === '443')) {
+            normalized.port = '';
+        }
+        if (!normalized.pathname) {
+            normalized.pathname = '/';
+        }
+        const sortedParams = [...normalized.searchParams.entries()]
+            .sort(([a], [b]) => a.localeCompare(b));
+        normalized.search = '';
+        for (const [key, value] of sortedParams) {
+            normalized.searchParams.append(key, value);
+        }
+        return normalized.toString();
+    }
+    catch {
+        return url.trim();
+    }
+}
+function getConditionalValidators(url) {
+    const key = normalizeUrlForConditionalCache(url);
+    const existing = conditionalValidatorsByUrl.get(key);
+    if (!existing) {
+        return null;
+    }
+    // LRU touch
+    conditionalValidatorsByUrl.delete(key);
+    conditionalValidatorsByUrl.set(key, existing);
+    return existing;
+}
+function setConditionalValidators(url, validators) {
+    const key = normalizeUrlForConditionalCache(url);
+    if (conditionalValidatorsByUrl.has(key)) {
+        conditionalValidatorsByUrl.delete(key);
+    }
+    conditionalValidatorsByUrl.set(key, validators);
+    while (conditionalValidatorsByUrl.size > CONDITIONAL_CACHE_MAX_ENTRIES) {
+        const oldestKey = conditionalValidatorsByUrl.keys().next().value;
+        if (!oldestKey) {
+            break;
+        }
+        conditionalValidatorsByUrl.delete(oldestKey);
+    }
+}
+function rememberConditionalValidators(url, response) {
+    const etag = response.headers.get('etag') || undefined;
+    const lastModified = response.headers.get('last-modified') || undefined;
+    if (!etag && !lastModified) {
+        return;
+    }
+    setConditionalValidators(url, { etag, lastModified });
+}
+function hasHeader(headers, name) {
+    const lowered = name.toLowerCase();
+    return Object.keys(headers).some((header) => header.toLowerCase() === lowered);
+}
+function getCachedResultFor304(url, fallbackUrl) {
+    const cached = getCached(url) || (fallbackUrl ? getCached(fallbackUrl) : null);
+    if (!cached) {
+        return null;
+    }
+    return {
+        html: cached.html,
+        buffer: cached.buffer,
+        url: cached.url || url,
+        statusCode: 304,
+        contentType: cached.contentType,
+        screenshot: cached.screenshot,
+    };
+}
+function createAbortError() {
+    const error = new Error('Operation aborted');
+    error.name = 'AbortError';
+    return error;
+}
 /**
  * SECURITY: Validate URL to prevent SSRF attacks
  * Blocks localhost, private IPs, link-local, and various bypass techniques
@@ -229,20 +333,37 @@ function validateUserAgent(userAgent) {
  * Fast and lightweight, but can be blocked by Cloudflare/bot detection
  * SECURITY: Manual redirect handling with SSRF re-validation
  */
-export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeaders) {
+export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeaders, abortSignal) {
     // SECURITY: Validate URL to prevent SSRF
     validateUrl(url);
+    if (abortSignal?.aborted) {
+        throw createAbortError();
+    }
     // Validate user agent if provided
-    const validatedUserAgent = userAgent ? validateUserAgent(userAgent) : getRandomUserAgent();
+    // SEC.gov requires a User-Agent with contact info (their documented automated access policy)
+    const hostname = new URL(url).hostname.toLowerCase();
+    const isSecGov = hostname === 'sec.gov' || hostname.endsWith('.sec.gov');
+    const validatedUserAgent = isSecGov
+        ? 'WebPeel/1.0 (support@webpeel.dev)'
+        : (userAgent ? validateUserAgent(userAgent) : getRandomUserAgent());
     // SECURITY: Merge custom headers with defaults, block Host header override
     const defaultHeaders = {
         'User-Agent': validatedUserAgent,
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
         'Accept-Language': 'en-US,en;q=0.9',
-        'Accept-Encoding': 'gzip, deflate, br',
+        'Accept-Encoding': 'br, gzip, deflate',
         'DNT': '1',
         'Connection': 'keep-alive',
         'Upgrade-Insecure-Requests': '1',
+        'Sec-CH-UA': '"Chromium";v="131", "Not_A Brand";v="24"',
+        'Sec-CH-UA-Mobile': '?0',
+        'Sec-CH-UA-Platform': '"macOS"',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-User': '?1',
+        'Cache-Control': 'max-age=0',
+        'Priority': 'u=0, i',
     };
     const mergedHeaders = { ...defaultHeaders };
     if (customHeaders) {
@@ -258,6 +379,15 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
     let redirectCount = 0;
     let currentUrl = url;
     const seenUrls = new Set();
+    try {
+        const hostname = new URL(url).hostname;
+        void resolveAndCache(hostname).catch(() => {
+            // Best-effort optimization only.
+        });
+    }
+    catch {
+        // Ignore URL parsing errors here; validation handles invalid input below.
+    }
     while (redirectCount <= MAX_REDIRECTS) {
         // Detect redirect loops
         if (seenUrls.has(currentUrl)) {
@@ -266,15 +396,34 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
         seenUrls.add(currentUrl);
         // Re-validate on each redirect
         validateUrl(currentUrl);
-        const controller = new AbortController();
-        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        const timeoutController = new AbortController();
+        const timer = setTimeout(() => timeoutController.abort(), timeoutMs);
+        const signal = abortSignal
+            ? AbortSignal.any([timeoutController.signal, abortSignal])
+            : timeoutController.signal;
         try {
+            const requestHeaders = { ...mergedHeaders };
+            const validators = getConditionalValidators(currentUrl);
+            if (validators?.etag && !hasHeader(requestHeaders, 'if-none-match')) {
+                requestHeaders['If-None-Match'] = validators.etag;
+            }
+            if (validators?.lastModified && !hasHeader(requestHeaders, 'if-modified-since')) {
+                requestHeaders['If-Modified-Since'] = validators.lastModified;
+            }
             const response = await undiciFetch(currentUrl, {
-                headers: mergedHeaders,
-                signal: controller.signal,
+                headers: requestHeaders,
+                signal,
+                dispatcher: httpPool,
                 redirect: 'manual', // SECURITY: Manual redirect handling
             });
             clearTimeout(timer);
+            if (response.status === 304) {
+                const cachedResult = getCachedResultFor304(currentUrl, url);
+                if (cachedResult) {
+                    return cachedResult;
+                }
+                throw new NetworkError('HTTP 304 received but no cached response is available');
+            }
             // Handle redirects manually
             if (response.status >= 300 && response.status < 400) {
                 const location = response.headers.get('location');
@@ -283,6 +432,15 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
                 }
                 // Resolve relative URLs
                 currentUrl = new URL(location, currentUrl).href;
+                try {
+                    const hostname = new URL(currentUrl).hostname;
+                    void resolveAndCache(hostname).catch(() => {
+                        // Best-effort optimization only.
+                    });
+                }
+                catch {
+                    // Ignore URL parsing errors here; validation handles invalid input below.
+                }
                 redirectCount++;
                 continue;
             }
@@ -292,20 +450,37 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
                 }
                 throw new NetworkError(`HTTP ${response.status}: ${response.statusText}`);
             }
-            // Content-Type detection — accept a wide range of text-based content
+            rememberConditionalValidators(currentUrl, response);
+            // Content-Type detection
             const contentType = response.headers.get('content-type') || '';
+            const contentTypeLower = contentType.toLowerCase();
+            const urlLower = currentUrl.toLowerCase();
+            // Support binary documents (PDF/DOCX) in the simple HTTP path.
+            const isPdf = contentTypeLower.includes('application/pdf') || urlLower.endsWith('.pdf');
+            const isDocx = contentTypeLower.includes('application/vnd.openxmlformats-officedocument.wordprocessingml.document') || urlLower.endsWith('.docx');
+            const isBinaryDoc = isPdf || isDocx;
+            // Accept a wide range of text-based content, plus supported binary documents.
             const ALLOWED_TYPES = [
-                'text/html', 'application/xhtml+xml', 'application/pdf',
+                'text/html', 'application/xhtml+xml',
                 'text/plain', 'text/markdown', 'text/csv',
                 'application/json', 'text/json',
                 'text/xml', 'application/xml', 'application/rss+xml', 'application/atom+xml',
                 'application/javascript', 'text/javascript', 'text/css',
+                // Documents
+                'application/pdf',
+                'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
             ];
-            const isAllowed = ALLOWED_TYPES.some(t => contentType.includes(t)) || !contentType;
+            const isAllowed = !contentTypeLower ||
+                ALLOWED_TYPES.some(t => contentTypeLower.includes(t)) ||
+                // Many servers mislabel docs as octet-stream; allow when URL implies a supported document.
+                (contentTypeLower.includes('application/octet-stream') && isBinaryDoc);
             if (!isAllowed) {
                 // Check if it's at least text-based
-                if (!contentType.startsWith('text/') && !contentType.includes('json') && !contentType.includes('xml')) {
-                    throw new WebPeelError(`Binary content type: ${contentType}. WebPeel handles text-based content only.`);
+                const isTexty = contentTypeLower.startsWith('text/') ||
+                    contentTypeLower.includes('json') ||
+                    contentTypeLower.includes('xml');
+                if (!isTexty) {
+                    throw new WebPeelError(`Binary content type: ${contentType}. WebPeel handles text-based content and PDF/DOCX documents only.`);
                 }
             }
             // SECURITY: Stream response with size limit (prevent memory exhaustion)
@@ -339,14 +514,18 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
                 combined.set(chunk, offset);
                 offset += chunk.length;
             }
-            const html = new TextDecoder().decode(combined);
+            const buffer = Buffer.from(combined);
+            const html = isBinaryDoc ? '' : new TextDecoder().decode(combined);
             // For HTML content, check for suspiciously small responses (bot blocks)
             // Non-HTML content (JSON, text, XML) can legitimately be short
-            const isHtmlContent = contentType.includes('html') || contentType.includes('xhtml');
+            const isHtmlContent = !isBinaryDoc && (contentTypeLower.includes('html') || contentTypeLower.includes('xhtml'));
             if (isHtmlContent && (!html || html.length < 100)) {
                 throw new BlockedError('Empty or suspiciously small response. Site may require JavaScript.');
             }
-            if (!html) {
+            if (!isBinaryDoc && !html) {
+                throw new NetworkError('Empty response body');
+            }
+            if (isBinaryDoc && buffer.length === 0) {
                 throw new NetworkError('Empty response body');
             }
             // Check for Cloudflare challenge (only relevant for HTML)
@@ -355,6 +534,7 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
             }
             return {
                 html,
+                buffer: isBinaryDoc ? buffer : undefined,
                 url: currentUrl,
                 statusCode: response.status,
                 contentType,
@@ -366,6 +546,9 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
                 throw error;
             }
             if (error instanceof Error && error.name === 'AbortError') {
+                if (abortSignal?.aborted && !timeoutController.signal.aborted) {
+                    throw createAbortError();
+                }
                 throw new TimeoutError(`Request timed out after ${timeoutMs}ms`);
             }
             // Provide specific error messages based on the actual cause
@@ -393,15 +576,101 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
     }
     throw new WebPeelError(`Too many redirects (max ${MAX_REDIRECTS})`);
 }
+export async function closePool() {
+    const oldPool = httpPool;
+    httpPool = createHttpPool();
+    await oldPool.close().catch(() => { });
+}
 let sharedBrowser = null;
 let sharedStealthBrowser = null;
 let activePagesCount = 0;
 const MAX_CONCURRENT_PAGES = 5;
+const PAGE_POOL_SIZE = 3;
+const pooledPages = new Set();
+const idlePagePool = [];
+let pagePoolFillPromise = null;
+function removePooledPage(page) {
+    pooledPages.delete(page);
+    const idleIndex = idlePagePool.indexOf(page);
+    if (idleIndex >= 0) {
+        idlePagePool.splice(idleIndex, 1);
+    }
+}
+function takePooledPage() {
+    while (idlePagePool.length > 0) {
+        const page = idlePagePool.shift();
+        if (page.isClosed()) {
+            removePooledPage(page);
+            continue;
+        }
+        return page;
+    }
+    return null;
+}
+async function ensurePagePool(browser) {
+    const activeBrowser = browser ?? sharedBrowser;
+    if (!activeBrowser || !activeBrowser.isConnected()) {
+        return;
+    }
+    if (pagePoolFillPromise) {
+        await pagePoolFillPromise;
+        return;
+    }
+    pagePoolFillPromise = (async () => {
+        while (pooledPages.size < PAGE_POOL_SIZE) {
+            const pooledPage = await activeBrowser.newPage({
+                userAgent: getRandomUserAgent(),
+            });
+            pooledPages.add(pooledPage);
+            idlePagePool.push(pooledPage);
+        }
+    })().finally(() => {
+        pagePoolFillPromise = null;
+    });
+    await pagePoolFillPromise;
+}
+async function recyclePooledPage(page) {
+    if (!pooledPages.has(page)) {
+        await page.close().catch(() => { });
+        return;
+    }
+    if (page.isClosed()) {
+        removePooledPage(page);
+        if (sharedBrowser?.isConnected()) {
+            void ensurePagePool(sharedBrowser).catch(() => { });
+        }
+        return;
+    }
+    try {
+        await page.unroute('**/*').catch(() => { });
+        await page.context().clearCookies().catch(() => { });
+        await page.setExtraHTTPHeaders({});
+        await page.goto('about:blank', { waitUntil: 'domcontentloaded', timeout: 5000 }).catch(() => { });
+        if (!idlePagePool.includes(page)) {
+            idlePagePool.push(page);
+        }
+    }
+    catch {
+        removePooledPage(page);
+        await page.close().catch(() => { });
+    }
+    if (sharedBrowser?.isConnected() && pooledPages.size < PAGE_POOL_SIZE) {
+        void ensurePagePool(sharedBrowser).catch(() => { });
+    }
+}
+export async function warmup() {
+    startDnsWarmup();
+    const browser = await getBrowser();
+    await ensurePagePool(browser);
+}
 async function getBrowser() {
     // SECURITY: Check if browser is still connected and healthy
     if (sharedBrowser) {
         try {
             if (sharedBrowser.isConnected()) {
+                if (pooledPages.size < PAGE_POOL_SIZE) {
+                    void ensurePagePool(sharedBrowser).catch(() => { });
+                }
                 return sharedBrowser;
             }
         }
@@ -410,7 +679,11 @@ async function getBrowser() {
             sharedBrowser = null;
         }
     }
+    pooledPages.clear();
+    idlePagePool.length = 0;
+    pagePoolFillPromise = null;
     sharedBrowser = await chromium.launch({ headless: true });
+    void ensurePagePool(sharedBrowser).catch(() => { });
     return sharedBrowser;
 }
 async function getStealthBrowser() {
@@ -436,13 +709,16 @@ async function getStealthBrowser() {
 export async function browserFetch(url, options = {}) {
     // SECURITY: Validate URL to prevent SSRF
     validateUrl(url);
-    const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, stealth = false, actions, keepPageOpen = false, } = options;
+    const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, stealth = false, actions, keepPageOpen = false, signal, } = options;
     // Validate user agent if provided
     const validatedUserAgent = userAgent ? validateUserAgent(userAgent) : getRandomUserAgent();
     // Validate wait time
     if (waitMs < 0 || waitMs > 60000) {
         throw new WebPeelError('Wait time must be between 0 and 60000ms');
     }
+    if (signal?.aborted) {
+        throw createAbortError();
+    }
     // SECURITY: Validate custom headers if provided
     if (headers) {
         for (const [key, value] of Object.entries(headers)) {
@@ -466,14 +742,51 @@ export async function browserFetch(url, options = {}) {
     }
     activePagesCount++;
     let page = null;
+    let usingPooledPage = false;
+    let abortHandler;
     try {
         const browser = stealth ? await getStealthBrowser() : await getBrowser();
-        page = await browser.newPage({
-            userAgent: validatedUserAgent,
-        });
-        // Set custom headers if provided
-        if (headers && Object.keys(headers).length > 0) {
-            await page.setExtraHTTPHeaders(headers);
+        const shouldUsePagePool = !stealth && !userAgent && !keepPageOpen;
+        if (shouldUsePagePool) {
+            page = takePooledPage();
+            usingPooledPage = !!page;
+            if (usingPooledPage && pooledPages.size < PAGE_POOL_SIZE) {
+                void ensurePagePool(browser).catch(() => { });
+            }
+        }
+        if (!page) {
+            const pageOptions = {
+                userAgent: validatedUserAgent,
+                ...(stealth
+                    ? {
+                        viewport: { width: 1920, height: 1080 },
+                        locale: 'en-US',
+                        timezoneId: 'America/New_York',
+                        javaScriptEnabled: true,
+                    }
+                    : {}),
+            };
+            page = await browser.newPage(pageOptions);
+            usingPooledPage = false;
+        }
+        else {
+            await page.setViewportSize({ width: 1280, height: 720 }).catch(() => { });
+        }
+        if (signal) {
+            abortHandler = () => {
+                if (page && !page.isClosed()) {
+                    void page.close().catch(() => { });
+                }
+            };
+            signal.addEventListener('abort', abortHandler, { once: true });
+        }
+        await page.unroute('**/*').catch(() => { });
+        const mergedHeaders = { ...(headers || {}) };
+        if (usingPooledPage) {
+            mergedHeaders['User-Agent'] = validatedUserAgent;
+        }
+        if (usingPooledPage || Object.keys(mergedHeaders).length > 0) {
+            await page.setExtraHTTPHeaders(mergedHeaders);
         }
         // Set cookies if provided
         if (cookies && cookies.length > 0) {
@@ -491,8 +804,12 @@ export async function browserFetch(url, options = {}) {
             });
             await page.context().addCookies(parsedCookies);
         }
-        // Block images, fonts, and other heavy resources for speed (unless screenshot is requested)
-        if (!screenshot) {
+        if (signal?.aborted) {
+            throw createAbortError();
+        }
+        // Block images/fonts/etc for speed in non-stealth mode.
+        // In stealth mode, blocking common resources can be a bot-detection signal.
+        if (!screenshot && !stealth) {
             await page.route('**/*', (route) => {
                 const resourceType = route.request().resourceType();
                 if (['image', 'font', 'media', 'stylesheet'].includes(resourceType)) {
@@ -504,19 +821,48 @@ export async function browserFetch(url, options = {}) {
             });
         }
         else {
-            // For screenshots, allow all resources
+            // For screenshots and stealth mode, allow all resources
             await page.route('**/*', (route) => route.continue());
         }
         // SECURITY: Wrap entire operation in timeout
         let screenshotBuffer;
+        const throwIfAborted = () => {
+            if (signal?.aborted) {
+                throw createAbortError();
+            }
+        };
         const fetchPromise = (async () => {
-            await page.goto(url, {
+            const response = await page.goto(url, {
                 waitUntil: 'domcontentloaded',
                 timeout: timeoutMs,
             });
-            // Wait for additional time if requested (for dynamic content)
+            throwIfAborted();
+            // Quick check: if body text is very thin, wait for JS to render more content.
+            // Only adds latency when the page clearly hasn't loaded yet.
+            // eslint-disable-next-line @typescript-eslint/no-implied-eval
+            const bodyTextLength = await page.evaluate('document.body?.innerText?.trim().length || 0').catch(() => 0);
+            if (bodyTextLength < 500) {
+                await page.waitForLoadState('networkidle', { timeout: 1500 }).catch(() => { });
+                throwIfAborted();
+            }
+            const finalUrl = page.url();
+            const contentType = response?.headers()?.['content-type'] || '';
+            const contentTypeLower = contentType.toLowerCase();
+            const urlLower = finalUrl.toLowerCase();
+            const isPdf = contentTypeLower.includes('application/pdf') || urlLower.endsWith('.pdf');
+            const isDocx = contentTypeLower.includes('wordprocessingml.document') || urlLower.endsWith('.docx');
+            const isBinaryDoc = !!response && (isPdf || isDocx);
+            // Small randomized delay in stealth mode (simulate human behavior)
+            // Keep it short — enough to look human, not enough to kill latency
+            if (stealth) {
+                const extraDelayMs = 200 + Math.floor(Math.random() * 601);
+                await page.waitForTimeout(extraDelayMs);
+                throwIfAborted();
+            }
+            // Wait for additional time if requested (for dynamic content / screenshots)
             if (waitMs > 0) {
                 await page.waitForTimeout(waitMs);
+                throwIfAborted();
             }
             // Execute page actions if provided
             if (actions && actions.length > 0) {
@@ -525,23 +871,59 @@ export async function browserFetch(url, options = {}) {
                 if (actionScreenshot) {
                     screenshotBuffer = actionScreenshot;
                 }
+                throwIfAborted();
+            }
+            // If the navigation returned a binary document (PDF/DOCX), grab the raw body.
+            if (isBinaryDoc) {
+                const buffer = await response.body();
+                throwIfAborted();
+                // Capture screenshot if requested (and not already captured by actions)
+                if (screenshot && !screenshotBuffer) {
+                    screenshotBuffer = await page.screenshot({
+                        fullPage: screenshotFullPage,
+                        type: 'png',
+                    });
+                }
+                return {
+                    html: '',
+                    finalUrl,
+                    buffer,
+                    contentType,
+                    statusCode: response.status(),
+                };
             }
             const html = await page.content();
-            const finalUrl = page.url();
-            return { html, finalUrl };
+            throwIfAborted();
+            return {
+                html,
+                finalUrl,
+                contentType,
+                statusCode: response?.status(),
+            };
         })();
+        let operationTimeout;
         const timeoutPromise = new Promise((_, reject) => {
-            setTimeout(() => reject(new TimeoutError(`Operation timed out after ${timeoutMs}ms`)), timeoutMs);
+            operationTimeout = setTimeout(() => reject(new TimeoutError(`Operation timed out after ${timeoutMs}ms`)), timeoutMs);
         });
-        const { html, finalUrl } = await Promise.race([fetchPromise, timeoutPromise]);
-        // SECURITY: Limit HTML size
-        if (html.length > 10 * 1024 * 1024) { // 10MB limit
-            throw new WebPeelError('Response too large (max 10MB)');
+        const fetchData = await Promise.race([fetchPromise, timeoutPromise]);
+        if (operationTimeout) {
+            clearTimeout(operationTimeout);
         }
-        if (!html || html.length < 100) {
-            throw new BlockedError('Empty or suspiciously small response from browser.');
+        const { html, finalUrl } = fetchData;
+        const fetchBuffer = 'buffer' in fetchData ? fetchData.buffer : undefined;
+        const fetchContentType = 'contentType' in fetchData ? fetchData.contentType : undefined;
+        const fetchStatusCode = 'statusCode' in fetchData ? fetchData.statusCode : undefined;
+        const isBinaryDoc = !!fetchBuffer;
+        // SECURITY: Limit HTML size (skip for binary documents where html is empty)
+        if (!isBinaryDoc) {
+            if (html.length > 10 * 1024 * 1024) { // 10MB limit
+                throw new WebPeelError('Response too large (max 10MB)');
+            }
+            if (!html || html.length < 100) {
+                throw new BlockedError('Empty or suspiciously small response from browser.');
+            }
         }
-        // Capture screenshot if requested (and not already captured by actions)
+        // Capture screenshot if requested (and not already captured by actions or document handler)
         if (screenshot && !screenshotBuffer) {
             screenshotBuffer = await page.screenshot({
                 fullPage: screenshotFullPage,
@@ -552,7 +934,10 @@ export async function browserFetch(url, options = {}) {
         if (keepPageOpen && page) {
             return {
                 html,
+                buffer: fetchBuffer,
                 url: finalUrl,
+                statusCode: fetchStatusCode,
+                contentType: fetchContentType,
                 screenshot: screenshotBuffer,
                 page,
                 browser,
@@ -560,7 +945,10 @@ export async function browserFetch(url, options = {}) {
         }
         return {
             html,
+            buffer: fetchBuffer,
             url: finalUrl,
+            statusCode: fetchStatusCode,
+            contentType: fetchContentType,
             screenshot: screenshotBuffer,
         };
     }
@@ -568,15 +956,26 @@ export async function browserFetch(url, options = {}) {
         if (error instanceof BlockedError || error instanceof WebPeelError || error instanceof TimeoutError) {
             throw error;
         }
+        if (error instanceof Error && error.name === 'AbortError') {
+            throw error;
+        }
         if (error instanceof Error && error.message.includes('Timeout')) {
             throw new TimeoutError(`Browser navigation timed out`);
         }
         throw new NetworkError(`Browser fetch failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
     }
     finally {
-        // CRITICAL: Always close page and decrement counter (unless keepPageOpen and no error)
+        if (signal && abortHandler) {
+            signal.removeEventListener('abort', abortHandler);
+        }
+        // CRITICAL: Always release/close page and decrement counter (unless keepPageOpen and no error)
         if (page && !keepPageOpen) {
-            await page.close().catch(() => { });
+            if (usingPooledPage) {
+                await recyclePooledPage(page);
+            }
+            else {
+                await page.close().catch(() => { });
+            }
         }
         activePagesCount--;
     }
@@ -584,6 +983,168 @@ export async function browserFetch(url, options = {}) {
 /**
  * Retry a fetch operation with exponential backoff
  */
+export async function browserScreenshot(url, options = {}) {
+    // SECURITY: Validate URL to prevent SSRF
+    validateUrl(url);
+    const { fullPage = false, width, height, format = 'png', quality, waitMs = 0, timeoutMs = 30000, userAgent, headers, cookies, stealth = false, actions, } = options;
+    const validatedUserAgent = userAgent ? validateUserAgent(userAgent) : getRandomUserAgent();
+    // Basic validation
+    if (waitMs < 0 || waitMs > 60000) {
+        throw new WebPeelError('Wait time must be between 0 and 60000ms');
+    }
+    if (timeoutMs < 1000 || timeoutMs > 120000) {
+        throw new WebPeelError('Timeout must be between 1000 and 120000ms');
+    }
+    if (width !== undefined && (!Number.isFinite(width) || width < 100 || width > 5000)) {
+        throw new WebPeelError('Width must be between 100 and 5000');
+    }
+    if (height !== undefined && (!Number.isFinite(height) || height < 100 || height > 5000)) {
+        throw new WebPeelError('Height must be between 100 and 5000');
+    }
+    if (format !== 'png' && format !== 'jpeg') {
+        throw new WebPeelError('Format must be png or jpeg');
+    }
+    if (format === 'jpeg' && quality !== undefined) {
+        if (!Number.isFinite(quality) || quality < 1 || quality > 100) {
+            throw new WebPeelError('JPEG quality must be between 1 and 100');
+        }
+    }
+    // SECURITY: Validate custom headers if provided
+    if (headers) {
+        for (const [key, value] of Object.entries(headers)) {
+            if (key.toLowerCase() === 'host') {
+                throw new WebPeelError('Custom Host header is not allowed');
+            }
+            if (typeof value !== 'string' || value.length > 500) {
+                throw new WebPeelError('Invalid header value');
+            }
+        }
+    }
+    // SECURITY: Limit concurrent browser pages with timeout
+    const queueStartTime = Date.now();
+    const QUEUE_TIMEOUT_MS = 30000;
+    while (activePagesCount >= MAX_CONCURRENT_PAGES) {
+        if (Date.now() - queueStartTime > QUEUE_TIMEOUT_MS) {
+            throw new TimeoutError('Browser page queue timeout - too many concurrent requests');
+        }
+        await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    activePagesCount++;
+    let page = null;
+    let usingPooledPage = false;
+    try {
+        const browser = stealth ? await getStealthBrowser() : await getBrowser();
+        const shouldUsePagePool = !stealth && !userAgent;
+        if (shouldUsePagePool) {
+            page = takePooledPage();
+            usingPooledPage = !!page;
+            if (usingPooledPage && pooledPages.size < PAGE_POOL_SIZE) {
+                void ensurePagePool(browser).catch(() => { });
+            }
+        }
+        if (!page) {
+            page = await browser.newPage({
+                userAgent: validatedUserAgent,
+                viewport: width || height ? {
+                    width: width || 1280,
+                    height: height || 720,
+                } : undefined,
+            });
+            usingPooledPage = false;
+        }
+        else {
+            await page.setViewportSize({
+                width: width || 1280,
+                height: height || 720,
+            }).catch(() => { });
+        }
+        await page.unroute('**/*').catch(() => { });
+        const mergedHeaders = { ...(headers || {}) };
+        if (usingPooledPage) {
+            mergedHeaders['User-Agent'] = validatedUserAgent;
+        }
+        if (usingPooledPage || Object.keys(mergedHeaders).length > 0) {
+            await page.setExtraHTTPHeaders(mergedHeaders);
+        }
+        if (cookies && cookies.length > 0) {
+            const parsedCookies = cookies.map(cookie => {
+                const [nameValue] = cookie.split(';').map(s => s.trim());
+                const [name, value] = nameValue.split('=');
+                if (!name || value === undefined) {
+                    throw new WebPeelError(`Invalid cookie format: ${cookie}`);
+                }
+                return {
+                    name: name.trim(),
+                    value: value.trim(),
+                    url,
+                };
+            });
+            await page.context().addCookies(parsedCookies);
+        }
+        // For screenshots, allow all resources
+        await page.route('**/*', (route) => route.continue());
+        let screenshotBuffer;
+        const doWork = (async () => {
+            await page.goto(url, {
+                waitUntil: 'domcontentloaded',
+                timeout: timeoutMs,
+            });
+            if (waitMs > 0) {
+                await page.waitForTimeout(waitMs);
+            }
+            if (actions && actions.length > 0) {
+                const { executeActions } = await import('./actions.js');
+                const actionScreenshot = await executeActions(page, actions, {
+                    fullPage,
+                    type: format,
+                    quality,
+                });
+                if (actionScreenshot) {
+                    screenshotBuffer = actionScreenshot;
+                }
+            }
+            const finalUrl = page.url();
+            // Capture screenshot if not captured via actions
+            if (!screenshotBuffer) {
+                screenshotBuffer = await page.screenshot({
+                    fullPage,
+                    type: format,
+                    ...(format === 'jpeg' && typeof quality === 'number' ? { quality } : {}),
+                });
+            }
+            return { finalUrl, screenshotBuffer: screenshotBuffer };
+        })();
+        let operationTimeout;
+        const timeoutPromise = new Promise((_, reject) => {
+            operationTimeout = setTimeout(() => reject(new TimeoutError(`Operation timed out after ${timeoutMs}ms`)), timeoutMs);
+        });
+        const { finalUrl, screenshotBuffer: buf } = await Promise.race([doWork, timeoutPromise]);
+        if (operationTimeout) {
+            clearTimeout(operationTimeout);
+        }
+        return { buffer: buf, finalUrl };
+    }
+    catch (error) {
+        if (error instanceof BlockedError || error instanceof WebPeelError || error instanceof TimeoutError) {
+            throw error;
+        }
+        if (error instanceof Error && error.message.includes('Timeout')) {
+            throw new TimeoutError('Browser screenshot timed out');
+        }
+        throw new NetworkError(`Browser screenshot failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+    finally {
+        if (page) {
+            if (usingPooledPage) {
+                await recyclePooledPage(page);
+            }
+            else {
+                await page.close().catch(() => { });
+            }
+        }
+        activePagesCount--;
+    }
+}
 export async function retryFetch(fn, maxAttempts = 3, baseDelayMs = 1000) {
     let lastError = null;
     for (let attempt = 1; attempt <= maxAttempts; attempt++) {
@@ -608,6 +1169,11 @@ export async function retryFetch(fn, maxAttempts = 3, baseDelayMs = 1000) {
  * Clean up browser resources
  */
 export async function cleanup() {
+    const pagesToClose = Array.from(pooledPages);
+    pooledPages.clear();
+    idlePagePool.length = 0;
+    pagePoolFillPromise = null;
+    await Promise.all(pagesToClose.map((page) => page.close().catch(() => { })));
     if (sharedBrowser) {
         await sharedBrowser.close();
         sharedBrowser = null;
@@ -616,5 +1182,6 @@ export async function cleanup() {
         await sharedStealthBrowser.close();
         sharedStealthBrowser = null;
     }
+    await closePool().catch(() => { });
 }
 //# sourceMappingURL=fetcher.js.map