webpeel 0.21.7 → 0.21.8

package/dist/core/proxy-config.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Shared Webshare residential proxy configuration.
+ *
+ * WebPeel uses Webshare residential proxies (configured via env vars) to route
+ * requests through US residential IPs, bypassing datacenter IP blocks from
+ * DuckDuckGo, Amazon, BestBuy, CarGurus, and other sites with anti-bot detection.
+ *
+ * Proxy credentials are loaded from environment variables:
+ *   WEBSHARE_PROXY_HOST  — proxy hostname (e.g. p.webshare.io)
+ *   WEBSHARE_PROXY_PORT  — base port number (e.g. 10000)
+ *   WEBSHARE_PROXY_USER  — proxy username (without slot suffix)
+ *   WEBSHARE_PROXY_PASS  — proxy password
+ *   WEBSHARE_PROXY_SLOTS — number of available US residential slots
+ *
+ * With the Webshare backbone plan each US slot has its own port:
+ *   slot N → port (WEBSHARE_PROXY_PORT + N - 1), username: USER-US-N
+ */
+export interface ProxyConfig {
+    /** Proxy server URL in the format "http://host:port" */
+    server: string;
+    /** Proxy username (includes slot suffix, e.g. "user-US-42") */
+    username: string;
+    /** Proxy password */
+    password: string;
+}
+/**
+ * Get a random Webshare residential proxy config.
+ * Returns null if the proxy is not configured (env vars missing or slots = 0).
+ *
+ * Uses random slot selection across all available US slots for even load
+ * distribution — same approach as youtube.ts proxyRequestSlotted().
+ */
+export declare function getWebshareProxy(): ProxyConfig | null;
+/**
+ * Check if Webshare proxies are configured (env vars are present and non-empty).
+ * Does NOT guarantee the proxy is reachable — just that credentials are set.
+ */
+export declare function hasWebshareProxy(): boolean;
+/**
+ * Convert a ProxyConfig to a Playwright-compatible proxy object.
+ * Useful for passing directly to browser.newContext({ proxy: ... }).
+ */
+export declare function toPlaywrightProxy(config: ProxyConfig): {
+    server: string;
+    username: string;
+    password: string;
+};
+/**
+ * Get a random Webshare proxy as a fully-qualified URL string with embedded
+ * credentials. The format is: `http://username:password@host:port`
+ *
+ * Useful for passing to strategies.ts proxy option (which expects a URL string).
+ * Returns null if proxies are not configured.
+ */
+export declare function getWebshareProxyUrl(): string | null;

package/dist/core/proxy-config.js

@@ -0,0 +1,79 @@
+/**
+ * Shared Webshare residential proxy configuration.
+ *
+ * WebPeel uses Webshare residential proxies (configured via env vars) to route
+ * requests through US residential IPs, bypassing datacenter IP blocks from
+ * DuckDuckGo, Amazon, BestBuy, CarGurus, and other sites with anti-bot detection.
+ *
+ * Proxy credentials are loaded from environment variables:
+ *   WEBSHARE_PROXY_HOST  — proxy hostname (e.g. p.webshare.io)
+ *   WEBSHARE_PROXY_PORT  — base port number (e.g. 10000)
+ *   WEBSHARE_PROXY_USER  — proxy username (without slot suffix)
+ *   WEBSHARE_PROXY_PASS  — proxy password
+ *   WEBSHARE_PROXY_SLOTS — number of available US residential slots
+ *
+ * With the Webshare backbone plan each US slot has its own port:
+ *   slot N → port (WEBSHARE_PROXY_PORT + N - 1), username: USER-US-N
+ */
+/**
+ * Get a random Webshare residential proxy config.
+ * Returns null if the proxy is not configured (env vars missing or slots = 0).
+ *
+ * Uses random slot selection across all available US slots for even load
+ * distribution — same approach as youtube.ts proxyRequestSlotted().
+ */
+export function getWebshareProxy() {
+    const host = process.env.WEBSHARE_PROXY_HOST;
+    const user = process.env.WEBSHARE_PROXY_USER;
+    const pass = process.env.WEBSHARE_PROXY_PASS;
+    const basePort = parseInt(process.env.WEBSHARE_PROXY_PORT || '10000', 10);
+    const slots = parseInt(process.env.WEBSHARE_PROXY_SLOTS || '0', 10);
+    if (!host || !user || !pass || slots <= 0)
+        return null;
+    const slot = Math.floor(Math.random() * slots) + 1;
+    const port = basePort + slot - 1;
+    return {
+        server: `http://${host}:${port}`,
+        username: `${user}-US-${slot}`,
+        password: pass,
+    };
+}
+/**
+ * Check if Webshare proxies are configured (env vars are present and non-empty).
+ * Does NOT guarantee the proxy is reachable — just that credentials are set.
+ */
+export function hasWebshareProxy() {
+    return !!(process.env.WEBSHARE_PROXY_HOST &&
+        process.env.WEBSHARE_PROXY_USER &&
+        process.env.WEBSHARE_PROXY_PASS);
+}
+/**
+ * Convert a ProxyConfig to a Playwright-compatible proxy object.
+ * Useful for passing directly to browser.newContext({ proxy: ... }).
+ */
+export function toPlaywrightProxy(config) {
+    return {
+        server: config.server,
+        username: config.username,
+        password: config.password,
+    };
+}
+/**
+ * Get a random Webshare proxy as a fully-qualified URL string with embedded
+ * credentials. The format is: `http://username:password@host:port`
+ *
+ * Useful for passing to strategies.ts proxy option (which expects a URL string).
+ * Returns null if proxies are not configured.
+ */
+export function getWebshareProxyUrl() {
+    const config = getWebshareProxy();
+    if (!config)
+        return null;
+    try {
+        const url = new URL(config.server);
+        return `http://${encodeURIComponent(config.username)}:${encodeURIComponent(config.password)}@${url.host}`;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/core/search-provider.js

@@ -15,6 +15,7 @@
 import { fetch as undiciFetch } from 'undici';
 import { load } from 'cheerio';
 import { getStealthBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
+import { getWebshareProxy } from './proxy-config.js';
 import { createLogger } from './logger.js';
 const log = createLogger('search');
 function decodeHtmlEntities(input) {
@@ -236,10 +237,12 @@ export class StealthSearchProvider {
             const browser = await getStealthBrowser();
             const params = new URLSearchParams({ q: query });
             const url = `https://html.duckduckgo.com/html/?${params.toString()}`;
+            const proxy = getWebshareProxy();
             ctx = await browser.newContext({
                 userAgent: getRandomUserAgent(),
                 locale: 'en-US',
                 timezoneId: 'America/New_York',
+                ...(proxy ? { proxy: { server: proxy.server, username: proxy.username, password: proxy.password } } : {}),
             });
             const page = await ctx.newPage();
             await applyStealthScripts(page);
@@ -303,10 +306,12 @@ export class StealthSearchProvider {
             const browser = await getStealthBrowser();
             const params = new URLSearchParams({ q: query });
             const url = `https://www.bing.com/search?${params.toString()}`;
+            const proxy = getWebshareProxy();
             ctx = await browser.newContext({
                 userAgent: getRandomUserAgent(),
                 locale: 'en-US',
                 timezoneId: 'America/New_York',
+                ...(proxy ? { proxy: { server: proxy.server, username: proxy.username, password: proxy.password } } : {}),
             });
             const page = await ctx.newPage();
             await applyStealthScripts(page);
@@ -380,10 +385,12 @@ export class StealthSearchProvider {
             const browser = await getStealthBrowser();
             const params = new URLSearchParams({ q: query });
             const url = `https://www.ecosia.org/search?${params.toString()}`;
+            const proxy = getWebshareProxy();
             ctx = await browser.newContext({
                 userAgent: getRandomUserAgent(),
                 locale: 'en-US',
                 timezoneId: 'America/New_York',
+                ...(proxy ? { proxy: { server: proxy.server, username: proxy.username, password: proxy.password } } : {}),
             });
             const page = await ctx.newPage();
             await applyStealthScripts(page);

package/dist/core/strategies.js

@@ -10,6 +10,7 @@ import { simpleFetch, browserFetch, retryFetch } from './fetcher.js';
 import { getCached, setCached as setBasicCache } from './cache.js';
 import { resolveAndCache } from './dns-cache.js';
 import { BlockedError, NetworkError } from '../types.js';
+import { getWebshareProxyUrl } from './proxy-config.js';
 import { detectChallenge } from './challenge-detection.js';
 import { getStrategyHooks, } from './strategy-hooks.js';
 import { createLogger } from './logger.js';
@@ -310,10 +311,15 @@ async function fetchWithBrowserStrategy(url, options) {
 export async function smartFetch(url, options = {}) {
     const { forceBrowser = false, stealth = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, actions, keepPageOpen = false, noCache = false, raceTimeoutMs = 2000, profileDir, headed = false, storageState, proxy, proxies, device, viewportWidth, viewportHeight, waitUntil, waitSelector, blockResources, cloaked = false, cycle = false, tls = false, noEscalate = false, } = options;
     const usePeelTLS = tls || cycle;
-    // Build effective proxy list: explicit proxies array, or single proxy, or empty
+    // Build effective proxy list: explicit proxies array, or single proxy, or empty.
+    // When no explicit proxy is configured and Webshare is available, automatically
+    // add it as a fallback: try direct connection first (fast), then Webshare on block.
     const effectiveProxies = proxies?.length ? proxies :
         proxy ? [proxy] :
-            [undefined]; // undefined = direct connection (no proxy)
+            (() => {
+                const wsUrl = getWebshareProxyUrl();
+                return wsUrl ? [undefined, wsUrl] : [undefined];
+            })();
     const firstProxy = effectiveProxies[0];
     const hooks = getStrategyHooks();
     const fetchStartMs = Date.now();

package/dist/core/youtube.js

@@ -15,6 +15,7 @@ import { join } from 'node:path';
 import { fetchTranscript as ytpFetchTranscript } from 'youtube-transcript-plus';
 import { simpleFetch } from './fetcher.js';
 import { getBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
+import { hasWebshareProxy as _hasWebshareProxy } from './proxy-config.js';
 import { createLogger } from './logger.js';
 // ---------------------------------------------------------------------------
 // yt-dlp startup diagnostics
@@ -239,8 +240,10 @@ export function extractSummary(fullText) {
 // ---------------------------------------------------------------------------
 // Proxy-based InnerTube transcript extraction
 // ---------------------------------------------------------------------------
-// Webshare residential proxy config — reads from env vars on Render.
+// Webshare residential proxy config — reads from env vars via proxy-config.ts.
 // Locally, falls back to direct fetch (residential IP already works).
+// These constants are kept for use in proxyRequestSlotted() which does
+// low-level HTTP CONNECT tunneling (not Playwright-level proxy).
 const PROXY_HOST = process.env.WEBSHARE_PROXY_HOST || 'p.webshare.io';
 const PROXY_BASE_PORT = parseInt(process.env.WEBSHARE_PROXY_PORT || '10000', 10);
 const PROXY_USER = process.env.WEBSHARE_PROXY_USER || '';
@@ -249,7 +252,8 @@ const PROXY_PASS = process.env.WEBSHARE_PROXY_PASS || '';
 // slot N → port (PROXY_BASE_PORT + N - 1), username: USER-US-N
 const PROXY_MAX_US_SLOTS = parseInt(process.env.WEBSHARE_PROXY_SLOTS || '44744', 10);
 function isProxyConfigured() {
-    return !!(PROXY_USER && PROXY_PASS);
+    // Delegate to the shared proxy-config helper for consistency
+    return _hasWebshareProxy();
 }
 /**
  * Make an HTTP(S) request through the Webshare CONNECT proxy with a specific

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.7",
+  "version": "0.21.8",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",