webpeel 0.8.1 → 0.9.0

package/dist/core/site-search.js

@@ -0,0 +1,253 @@
+/**
+ * Site-Aware Search URL Builders
+ *
+ * Provides URL templates for popular websites so AI agents can search them
+ * without needing to know site-specific URL structures.
+ *
+ * @module site-search
+ */
+/**
+ * URL templates for popular sites, keyed by site ID.
+ * All query values are URL-encoded via encodeURIComponent.
+ */
+export const SITE_TEMPLATES = {
+    // ── Shopping ──────────────────────────────────────────────────────────────
+    ebay: {
+        name: 'eBay',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.ebay.com/sch/i.html?_nkw=${encodeURIComponent(q)}`,
+    },
+    amazon: {
+        name: 'Amazon',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.amazon.com/s?k=${encodeURIComponent(q)}`,
+    },
+    walmart: {
+        name: 'Walmart',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.walmart.com/search?q=${encodeURIComponent(q)}`,
+    },
+    target: {
+        name: 'Target',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.target.com/s?searchTerm=${encodeURIComponent(q)}`,
+    },
+    bestbuy: {
+        name: 'Best Buy',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.bestbuy.com/site/searchpage.jsp?st=${encodeURIComponent(q)}`,
+    },
+    etsy: {
+        name: 'Etsy',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.etsy.com/search?q=${encodeURIComponent(q)}`,
+    },
+    aliexpress: {
+        name: 'AliExpress',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.aliexpress.com/wholesale?SearchText=${encodeURIComponent(q)}`,
+    },
+    newegg: {
+        name: 'Newegg',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.newegg.com/p/pl?d=${encodeURIComponent(q)}`,
+    },
+    // ── General ───────────────────────────────────────────────────────────────
+    google: {
+        name: 'Google',
+        category: 'general',
+        searchUrl: (q) => `https://www.google.com/search?q=${encodeURIComponent(q)}`,
+    },
+    bing: {
+        name: 'Bing',
+        category: 'general',
+        searchUrl: (q) => `https://www.bing.com/search?q=${encodeURIComponent(q)}`,
+    },
+    duckduckgo: {
+        name: 'DuckDuckGo',
+        category: 'general',
+        searchUrl: (q) => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(q)}`,
+    },
+    // ── Social / Content ──────────────────────────────────────────────────────
+    reddit: {
+        name: 'Reddit',
+        category: 'social',
+        searchUrl: (q) => `https://www.reddit.com/search/?q=${encodeURIComponent(q)}`,
+    },
+    youtube: {
+        name: 'YouTube',
+        category: 'social',
+        searchUrl: (q) => `https://www.youtube.com/results?search_query=${encodeURIComponent(q)}`,
+    },
+    twitter: {
+        name: 'X (Twitter)',
+        category: 'social',
+        searchUrl: (q) => `https://x.com/search?q=${encodeURIComponent(q)}`,
+    },
+    linkedin: {
+        name: 'LinkedIn',
+        category: 'social',
+        searchUrl: (q) => `https://www.linkedin.com/search/results/all/?keywords=${encodeURIComponent(q)}`,
+    },
+    // ── Tech ──────────────────────────────────────────────────────────────────
+    github: {
+        name: 'GitHub',
+        category: 'tech',
+        searchUrl: (q) => `https://github.com/search?q=${encodeURIComponent(q)}`,
+    },
+    stackoverflow: {
+        name: 'Stack Overflow',
+        category: 'tech',
+        searchUrl: (q) => `https://stackoverflow.com/search?q=${encodeURIComponent(q)}`,
+    },
+    npm: {
+        name: 'npm',
+        category: 'tech',
+        searchUrl: (q) => `https://www.npmjs.com/search?q=${encodeURIComponent(q)}`,
+    },
+    pypi: {
+        name: 'PyPI',
+        category: 'tech',
+        searchUrl: (q) => `https://pypi.org/search/?q=${encodeURIComponent(q)}`,
+    },
+    // ── Real Estate ───────────────────────────────────────────────────────────
+    zillow: {
+        name: 'Zillow',
+        category: 'real-estate',
+        searchUrl: (q) => `https://www.zillow.com/homes/${encodeURIComponent(q)}_rb/`,
+    },
+    realtor: {
+        name: 'Realtor.com',
+        category: 'real-estate',
+        searchUrl: (q) => `https://www.realtor.com/realestateandhomes-search/${encodeURIComponent(q)}`,
+    },
+    // ── Jobs ──────────────────────────────────────────────────────────────────
+    indeed: {
+        name: 'Indeed',
+        category: 'jobs',
+        searchUrl: (q) => `https://www.indeed.com/jobs?q=${encodeURIComponent(q)}`,
+    },
+    glassdoor: {
+        name: 'Glassdoor',
+        category: 'jobs',
+        searchUrl: (q) => `https://www.glassdoor.com/Job/jobs.htm?sc.keyword=${encodeURIComponent(q)}`,
+    },
+    'linkedin-jobs': {
+        name: 'LinkedIn Jobs',
+        category: 'jobs',
+        searchUrl: (q) => `https://www.linkedin.com/jobs/search/?keywords=${encodeURIComponent(q)}`,
+    },
+    // ── Food ──────────────────────────────────────────────────────────────────
+    yelp: {
+        name: 'Yelp',
+        category: 'food',
+        searchUrl: (q) => `https://www.yelp.com/search?find_desc=${encodeURIComponent(q)}`,
+    },
+    doordash: {
+        name: 'DoorDash',
+        category: 'food',
+        searchUrl: (q) => `https://www.doordash.com/search/store/${encodeURIComponent(q)}`,
+    },
+    ubereats: {
+        name: 'Uber Eats',
+        category: 'food',
+        searchUrl: (q) => `https://www.ubereats.com/search?q=${encodeURIComponent(q)}`,
+    },
+};
+/** Aliases that map to canonical site IDs */
+const SITE_ALIASES = {
+    x: 'twitter',
+    'best-buy': 'bestbuy',
+    'ali-express': 'aliexpress',
+    'stack-overflow': 'stackoverflow',
+    'duck-duck-go': 'duckduckgo',
+};
+/**
+ * Resolve a site ID (or alias) to its canonical key.
+ * Returns null if not found.
+ */
+function resolveSiteId(site) {
+    const lower = site.toLowerCase();
+    if (lower in SITE_TEMPLATES)
+        return lower;
+    if (lower in SITE_ALIASES)
+        return SITE_ALIASES[lower];
+    return null;
+}
+/**
+ * Build a search URL for a given site and query.
+ *
+ * @param site  Site ID (e.g. "ebay", "amazon") or alias (e.g. "x")
+ * @param query Search query string
+ * @throws Error if the site is not recognized
+ */
+export function buildSiteSearchUrl(site, query) {
+    const canonical = resolveSiteId(site);
+    if (!canonical) {
+        const available = Object.keys(SITE_TEMPLATES).join(', ');
+        throw new Error(`Unknown site: "${site}". Available sites: ${available}. ` +
+            `Run "webpeel sites" to list all supported sites.`);
+    }
+    const template = SITE_TEMPLATES[canonical];
+    return {
+        url: template.searchUrl(query),
+        site: canonical,
+        query,
+    };
+}
+/**
+ * List all available site templates in a flat array.
+ */
+export function listSites() {
+    return Object.entries(SITE_TEMPLATES).map(([id, template]) => ({
+        id,
+        name: template.name,
+        category: template.category,
+    }));
+}
+/**
+ * Find which site ID a given URL belongs to (reverse lookup).
+ * Returns the canonical site ID, or null if the URL doesn't match any template.
+ */
+export function findSiteByUrl(url) {
+    try {
+        const parsed = new URL(url);
+        const hostname = parsed.hostname.replace(/^www\./, '');
+        // Map of hostnames to site IDs
+        const hostnameMap = {
+            'ebay.com': 'ebay',
+            'amazon.com': 'amazon',
+            'walmart.com': 'walmart',
+            'target.com': 'target',
+            'bestbuy.com': 'bestbuy',
+            'etsy.com': 'etsy',
+            'aliexpress.com': 'aliexpress',
+            'newegg.com': 'newegg',
+            'google.com': 'google',
+            'bing.com': 'bing',
+            'html.duckduckgo.com': 'duckduckgo',
+            'duckduckgo.com': 'duckduckgo',
+            'reddit.com': 'reddit',
+            'youtube.com': 'youtube',
+            'x.com': 'twitter',
+            'twitter.com': 'twitter',
+            'linkedin.com': 'linkedin',
+            'github.com': 'github',
+            'stackoverflow.com': 'stackoverflow',
+            'npmjs.com': 'npm',
+            'pypi.org': 'pypi',
+            'zillow.com': 'zillow',
+            'realtor.com': 'realtor',
+            'indeed.com': 'indeed',
+            'glassdoor.com': 'glassdoor',
+            'yelp.com': 'yelp',
+            'doordash.com': 'doordash',
+            'ubereats.com': 'ubereats',
+        };
+        return hostnameMap[hostname] ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=site-search.js.map

package/dist/core/site-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"site-search.js","sourceRoot":"","sources":["../../src/core/site-search.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAcH;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAiC;IAC1D,6EAA6E;IAC7E,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,wCAAwC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAClF;IACD,MAAM,EAAE;QACN,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,kBAAkB,CAAC,CAAC,CAAC,EAAE;KACxE;IACD,OAAO,EAAE;QACP,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,oCAAoC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC9E;IACD,MAAM,EAAE;QACN,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,uCAAuC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KACjF;IACD,OAAO,EAAE;QACP,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,kDAAkD,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC5F;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,iCAAiC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC3E;IACD,UAAU,EAAE;QACV,IAAI,EAAE,YAAY;QAClB,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,mDAAmD,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC7F;IACD,MAAM,EAAE;QACN,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,iCAAiC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC3E;IAED,6EAA6E;IAC7E,MAAM,EAAE;QACN,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,SAAS;QACnB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,mCAAmC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC7E;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,SAAS;QACnB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,iCAAiC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC3E;IACD,UAAU,EAAE;QACV,IAAI,EAAE,YAAY;QAClB,QAAQ,EAAE,SAAS;QACnB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,uCAAuC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KACjF;IAED,6EAA6E;IAC7E,MAAM,EAAE;QACN,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,QAAQ;QAClB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,oCAAoC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC9E;IACD,OAAO,EAAE;QACP,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,QAAQ;QAClB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,gDAAgD,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC1F;IACD,OAAO,EAAE;QACP,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,QAAQ;QAClB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,0BAA0B,kBAAkB,CAAC,CAAC,CAAC,EAAE;KACpE;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE,QAAQ;QAClB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,yDAAyD,kBAAkB,CAAC,CAAC,CAAC,EAAE;KACnG;IAED,6EAA6E;IAC7E,MAAM,EAAE;QACN,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,+BAA+B,kBAAkB,CAAC,CAAC,CAAC,EAAE;KACzE;IACD,aAAa,EAAE;QACb,IAAI,EAAE,gBAAgB;QACtB,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,sCAAsC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAChF;IACD,GAAG,EAAE;QACH,IAAI,EAAE,KAAK;QACX,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,kCAAkC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC5E;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,kBAAkB,CAAC,CAAC,CAAC,EAAE;KACxE;IAED,6EAA6E;IAC7E,MAAM,EAAE;QACN,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,aAAa;QACvB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,gCAAgC,kBAAkB,CAAC,CAAC,CAAC,MAAM;KAC9E;IACD,OAAO,EAAE;QACP,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,aAAa;QACvB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,qDAAqD,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC/F;IAED,6EAA6E;IAC7E,MAAM,EAAE;QACN,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,iCAAiC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC3E;IACD,SAAS,EAAE;QACT,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,qDAAqD,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC/F;IACD,eAAe,EAAE;QACf,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,kDAAkD,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC5F;IAED,6EAA6E;IAC7E,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,yCAAyC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KACnF;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,yCAAyC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KACnF;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,qCAAqC,kBAAkB,CAAC,CAAC,CAAC,EAAE;KAC/E;CACF,CAAC;AAEF,6CAA6C;AAC7C,MAAM,YAAY,GAA2B;IAC3C,CAAC,EAAE,SAAS;IACZ,UAAU,EAAE,SAAS;IACrB,aAAa,EAAE,YAAY;IAC3B,gBAAgB,EAAE,eAAe;IACjC,cAAc,EAAE,YAAY;CAC7B,CAAC;AAEF;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,IAAI,KAAK,IAAI,cAAc;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,KAAK,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC,KAAK,CAAE,CAAC;IACvD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAE,KAAa;IAC5D,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzD,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,uBAAuB,SAAS,IAAI;YAC1D,kDAAkD,CACnD,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAE,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC;QAC9B,IAAI,EAAE,SAAS;QACf,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS;IACvB,OAAO,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7D,EAAE;QACF,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;KAC5B,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAEvD,+BAA+B;QAC/B,MAAM,WAAW,GAA2B;YAC1C,UAAU,EAAE,MAAM;YAClB,YAAY,EAAE,QAAQ;YACtB,aAAa,EAAE,SAAS;YACxB,YAAY,EAAE,QAAQ;YACtB,aAAa,EAAE,SAAS;YACxB,UAAU,EAAE,MAAM;YAClB,gBAAgB,EAAE,YAAY;YAC9B,YAAY,EAAE,QAAQ;YACtB,YAAY,EAAE,QAAQ;YACtB,UAAU,EAAE,MAAM;YAClB,qBAAqB,EAAE,YAAY;YACnC,gBAAgB,EAAE,YAAY;YAC9B,YAAY,EAAE,QAAQ;YACtB,aAAa,EAAE,SAAS;YACxB,OAAO,EAAE,SAAS;YAClB,aAAa,EAAE,SAAS;YACxB,cAAc,EAAE,UAAU;YAC1B,YAAY,EAAE,QAAQ;YACtB,mBAAmB,EAAE,eAAe;YACpC,WAAW,EAAE,KAAK;YAClB,UAAU,EAAE,MAAM;YAClB,YAAY,EAAE,QAAQ;YACtB,aAAa,EAAE,SAAS;YACxB,YAAY,EAAE,QAAQ;YACtB,eAAe,EAAE,WAAW;YAC5B,UAAU,EAAE,MAAM;YAClB,cAAc,EAAE,UAAU;YAC1B,cAAc,EAAE,UAAU;SAC3B,CAAC;QAEF,OAAO,WAAW,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/core/strategies.d.ts

@@ -34,6 +34,14 @@ export interface StrategyOptions {
         country?: string;
         languages?: string[];
     };
+    /**
+     * Path to a persistent Chrome user-data-dir.
+     * When set, bypasses the shared browser pool so cookies/sessions survive
+     * between fetch calls in the same process.
+     */
+    profileDir?: string;
+    /** Launch browser in headed (visible) mode — useful for debugging and profile setup. */
+    headed?: boolean;
 }
 /**
  * Smart fetch with automatic escalation.

package/dist/core/strategies.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAoE1D,MAAM,WAAW,eAAe;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EACA,MAAM,GACN,OAAO,GACP,QAAQ,GACR,MAAM,GACN,MAAM,GACN,QAAQ,GACR,OAAO,GACP,OAAO,GACP,iBAAiB,GACjB,YAAY,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;QAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE;QACT,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACH;AAuGD;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC,CA6QzB;AAID;;GAEG;AACH,OAAO,EAAE,kBAAkB,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC"}
+{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAOH,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AA2H1D,MAAM,WAAW,eAAe;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EACA,MAAM,GACN,OAAO,GACP,QAAQ,GACR,MAAM,GACN,MAAM,GACN,QAAQ,GACR,OAAO,GACP,OAAO,GACP,iBAAiB,GACjB,YAAY,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;QAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE;QACT,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wFAAwF;IACxF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAiHD;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC,CAiWzB;AAID;;GAEG;AACH,OAAO,EAAE,kBAAkB,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/core/strategies.js

@@ -10,30 +10,70 @@ import { simpleFetch, browserFetch, retryFetch } from './fetcher.js';
 import { getCached, setCached as setBasicCache } from './cache.js';
 import { resolveAndCache } from './dns-cache.js';
 import { BlockedError, NetworkError } from '../types.js';
+import { detectChallenge } from './challenge-detection.js';
 import { getStrategyHooks, } from './strategy-hooks.js';
 /* ---------- hardcoded domain rules -------------------------------------- */
 function shouldForceBrowser(url) {
     try {
         const hostname = new URL(url).hostname.toLowerCase();
-        // Reddit often returns an HTML shell via simple fetch
-        if (hostname === 'reddit.com' || hostname.endsWith('.reddit.com')) {
-            return { mode: 'browser' };
-        }
-        // npmjs blocks simple fetch with 403 frequently
-        if (hostname === 'npmjs.com' ||
-            hostname === 'www.npmjs.com' ||
-            hostname.endsWith('.npmjs.com')) {
-            return { mode: 'browser' };
-        }
-        // These are known to aggressively block automation
-        if (hostname === 'glassdoor.com' || hostname.endsWith('.glassdoor.com')) {
-            return { mode: 'stealth' };
-        }
-        if (hostname === 'bloomberg.com' || hostname.endsWith('.bloomberg.com')) {
-            return { mode: 'stealth' };
+        // Sites that return HTML shells / need JS rendering (browser mode)
+        const browserDomains = [
+            'reddit.com', // HTML shell via simple fetch
+            'npmjs.com', // 403 on simple fetch
+            'x.com', // SPA, login wall
+            'twitter.com', // SPA, login wall
+            'instagram.com', // SPA, login wall
+            'facebook.com', // SPA, heavy JS
+            'tiktok.com', // SPA, JS-rendered
+            'pinterest.com', // SPA, JS-rendered
+            'airbnb.com', // heavy SPA
+            'medium.com', // JS-rendered, sometimes login wall
+            'substack.com', // JS-rendered
+            'notion.so', // SPA
+            'figma.com', // SPA
+            'canva.com', // SPA
+            'vercel.app', // Could be any SPA
+        ];
+        for (const domain of browserDomains) {
+            if (hostname === domain || hostname.endsWith(`.${domain}`)) {
+                return { mode: 'browser' };
+            }
         }
-        if (hostname === 'indeed.com' || hostname.endsWith('.indeed.com')) {
-            return { mode: 'stealth' };
+        // These are known to aggressively block automation — stealth mode required
+        const stealthDomains = [
+            'glassdoor.com',
+            'bloomberg.com',
+            'indeed.com',
+            'amazon.com', // captcha wall on simple/browser fetch
+            'zillow.com', // aggressive bot detection
+            'ticketmaster.com', // Distil Networks / PerimeterX
+            'stubhub.com', // PerimeterX / CAPTCHA
+            'walmart.com', // Akamai Bot Manager
+            'target.com', // Akamai Bot Manager
+            'bestbuy.com', // Akamai Bot Manager
+            'homedepot.com', // Akamai Bot Manager
+            'lowes.com', // Akamai Bot Manager
+            'costco.com', // Akamai Bot Manager
+            'nike.com', // Akamai / Shape Security
+            'footlocker.com', // PerimeterX / DataDome
+            'realtor.com', // aggressive bot detection
+            'redfin.com', // aggressive bot detection
+            'cloudflare.com', // Cloudflare challenge pages
+            'ebay.com', // challenge page on simple fetch
+            'linkedin.com', // aggressive bot detection + login walls
+            'craigslist.org', // occasionally blocks automated access
+            'etsy.com', // Akamai protection
+            'wayfair.com', // Akamai protection
+            'newegg.com', // bot detection
+            'zappos.com', // Amazon subsidiary, same protection
+            'chewy.com', // Amazon subsidiary
+            'aliexpress.com', // anti-bot
+            'wish.com', // anti-bot
+        ];
+        for (const domain of stealthDomains) {
+            if (hostname === domain || hostname.endsWith(`.${domain}`)) {
+                return { mode: 'stealth' };
+            }
         }
     }
     catch {
@@ -57,6 +97,25 @@ function looksLikeShellPage(result) {
     const text = result.html.replace(/<[^>]*>/g, '').trim();
     return text.length < 500 && result.html.length > 1000;
 }
+/**
+ * Detect pages that returned HTML but have very little actual text content.
+ * This catches JS-rendered SPAs that return a shell page with a big HTML payload
+ * (scripts, styles, framework boilerplate) but minimal visible text.
+ */
+function shouldEscalateForLowContent(result) {
+    const ct = (result.contentType || '').toLowerCase();
+    if (!ct.includes('html'))
+        return false;
+    if (result.html.length <= 1500)
+        return false;
+    // Strip script/style blocks and their contents first, then strip remaining tags
+    const withoutScripts = result.html
+        .replace(/<script[\s\S]*?<\/script>/gi, '')
+        .replace(/<style[\s\S]*?<\/style>/gi, '')
+        .replace(/<noscript[\s\S]*?<\/noscript>/gi, '');
+    const visibleText = withoutScripts.replace(/<[^>]*>/g, '').replace(/\s+/g, ' ').trim();
+    return visibleText.length < 200;
+}
 function prefetchDns(url) {
     try {
         const hostname = new URL(url).hostname;
@@ -67,7 +126,7 @@ function prefetchDns(url) {
     }
 }
 async function fetchWithBrowserStrategy(url, options) {
-    const { userAgent, waitMs, timeoutMs, screenshot, screenshotFullPage, headers, cookies, actions, keepPageOpen, effectiveStealth, signal, } = options;
+    const { userAgent, waitMs, timeoutMs, screenshot, screenshotFullPage, headers, cookies, actions, keepPageOpen, effectiveStealth, signal, profileDir, headed, } = options;
     try {
         const result = await browserFetch(url, {
             userAgent,
@@ -81,6 +140,8 @@ async function fetchWithBrowserStrategy(url, options) {
             actions,
             keepPageOpen,
             signal,
+            profileDir,
+            headed,
         });
         return {
             ...result,
@@ -104,6 +165,8 @@ async function fetchWithBrowserStrategy(url, options) {
                 actions,
                 keepPageOpen,
                 signal,
+                profileDir,
+                headed,
             });
             return { ...result, method: 'stealth' };
         }
@@ -122,6 +185,8 @@ async function fetchWithBrowserStrategy(url, options) {
                 actions,
                 keepPageOpen,
                 signal,
+                profileDir,
+                headed,
             });
             return { ...result, method: effectiveStealth ? 'stealth' : 'browser' };
         }
@@ -136,7 +201,7 @@ async function fetchWithBrowserStrategy(url, options) {
  * With premium hooks: SWR cache → domain intel → parallel race → escalation.
  */
 export async function smartFetch(url, options = {}) {
-    const { forceBrowser = false, stealth = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, actions, keepPageOpen = false, noCache = false, raceTimeoutMs = 2000, } = options;
+    const { forceBrowser = false, stealth = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, actions, keepPageOpen = false, noCache = false, raceTimeoutMs = 2000, profileDir, headed = false, } = options;
     const hooks = getStrategyHooks();
     const fetchStartMs = Date.now();
     const recordMethod = (method) => {
@@ -198,6 +263,10 @@ export async function smartFetch(url, options = {}) {
     }
     /* ---- browser-level options ------------------------------------------- */
     let shouldUseBrowser = effectiveForceBrowser || screenshot || effectiveStealth;
+    // A profileDir always forces browser mode (profile sessions need a real browser)
+    if (profileDir) {
+        effectiveForceBrowser = true;
+    }
     const browserOptions = {
         userAgent,
         waitMs,
@@ -209,6 +278,8 @@ export async function smartFetch(url, options = {}) {
         actions,
         keepPageOpen,
         effectiveStealth,
+        profileDir,
+        headed,
     };
     /* ---- Strategy: simple fetch (with optional race) --------------------- */
     if (!shouldUseBrowser) {
@@ -236,15 +307,29 @@ export async function smartFetch(url, options = {}) {
         if (raceTimer)
             clearTimeout(raceTimer);
         if (simpleOrTimeout.type === 'simple-success') {
-            const strategyResult = {
-                ...simpleOrTimeout.result,
-                method: 'simple',
-            };
-            if (canUseCache) {
-                hooks.setCache?.(url, strategyResult) ?? setBasicCache(url, strategyResult);
+            // Check if the content is suspiciously thin — escalate to browser if so
+            if (shouldEscalateForLowContent(simpleOrTimeout.result)) {
+                shouldUseBrowser = true;
+            }
+            else {
+                // Check whether the response is a bot-challenge page (e.g. Cloudflare, PerimeterX)
+                const challengeCheck = detectChallenge(simpleOrTimeout.result.html, simpleOrTimeout.result.statusCode);
+                if (challengeCheck.isChallenge && challengeCheck.confidence >= 0.7) {
+                    // Escalate — the browser/stealth path will handle it below
+                    shouldUseBrowser = true;
+                }
+                else {
+                    const strategyResult = {
+                        ...simpleOrTimeout.result,
+                        method: 'simple',
+                    };
+                    if (canUseCache) {
+                        hooks.setCache?.(url, strategyResult) ?? setBasicCache(url, strategyResult);
+                    }
+                    recordMethod('simple');
+                    return strategyResult;
+                }
             }
-            recordMethod('simple');
-            return strategyResult;
         }
         if (simpleOrTimeout.type === 'simple-error') {
             if (!shouldEscalateSimpleError(simpleOrTimeout.error)) {
@@ -317,30 +402,85 @@ export async function smartFetch(url, options = {}) {
                     .then((result) => ({ type: 'simple-success', result }))
                     .catch((error) => ({ type: 'simple-error', error }));
                 if (simpleResult.type === 'simple-success') {
-                    const strategyResult = {
-                        ...simpleResult.result,
-                        method: 'simple',
-                    };
-                    if (canUseCache) {
-                        hooks.setCache?.(url, strategyResult) ?? setBasicCache(url, strategyResult);
+                    // Check if the content is suspiciously thin — escalate to browser if so
+                    if (shouldEscalateForLowContent(simpleResult.result)) {
+                        shouldUseBrowser = true;
+                    }
+                    else {
+                        // Check whether the response is a bot-challenge page
+                        const challengeCheck = detectChallenge(simpleResult.result.html, simpleResult.result.statusCode);
+                        if (challengeCheck.isChallenge && challengeCheck.confidence >= 0.7) {
+                            shouldUseBrowser = true;
+                        }
+                        else {
+                            const strategyResult = {
+                                ...simpleResult.result,
+                                method: 'simple',
+                            };
+                            if (canUseCache) {
+                                hooks.setCache?.(url, strategyResult) ?? setBasicCache(url, strategyResult);
+                            }
+                            recordMethod('simple');
+                            return strategyResult;
+                        }
                     }
-                    recordMethod('simple');
-                    return strategyResult;
                 }
-                if (!shouldEscalateSimpleError(simpleResult.error)) {
-                    throw simpleResult.error;
+                else {
+                    if (!shouldEscalateSimpleError(simpleResult.error)) {
+                        throw simpleResult.error;
+                    }
+                    shouldUseBrowser = true;
                 }
-                shouldUseBrowser = true;
             }
         }
     }
-    /* ---- browser / stealth fallback -------------------------------------- */
-    const browserResult = await fetchWithBrowserStrategy(url, browserOptions);
-    if (canUseCache) {
-        hooks.setCache?.(url, browserResult) ?? setBasicCache(url, browserResult);
+    /* ---- browser / stealth fallback with challenge-detection cascade ----- */
+    // Attempt 1: browser (or stealth, if already forced)
+    let finalResult = await fetchWithBrowserStrategy(url, browserOptions);
+    // Check if the browser result is itself a bot-challenge page
+    const browserChallengeCheck = detectChallenge(finalResult.html, finalResult.statusCode);
+    if (browserChallengeCheck.isChallenge && browserChallengeCheck.confidence >= 0.7) {
+        if (!browserOptions.effectiveStealth) {
+            // Attempt 2: escalate to stealth
+            const stealthOptions = {
+                ...browserOptions,
+                effectiveStealth: true,
+            };
+            finalResult = await fetchWithBrowserStrategy(url, stealthOptions);
+            const stealthChallengeCheck = detectChallenge(finalResult.html, finalResult.statusCode);
+            if (stealthChallengeCheck.isChallenge && stealthChallengeCheck.confidence >= 0.7) {
+                // Attempt 3: stealth + 5s extra wait
+                const stealthExtraOptions = {
+                    ...stealthOptions,
+                    waitMs: stealthOptions.waitMs + 5000,
+                };
+                finalResult = await fetchWithBrowserStrategy(url, stealthExtraOptions);
+                const finalChallengeCheck = detectChallenge(finalResult.html, finalResult.statusCode);
+                if (finalChallengeCheck.isChallenge && finalChallengeCheck.confidence >= 0.7) {
+                    // Give up — return with warning flag
+                    finalResult = { ...finalResult, challengeDetected: true };
+                }
+            }
+        }
+        else {
+            // Already in stealth mode; retry with 5s extra wait
+            const stealthExtraOptions = {
+                ...browserOptions,
+                waitMs: browserOptions.waitMs + 5000,
+            };
+            finalResult = await fetchWithBrowserStrategy(url, stealthExtraOptions);
+            const finalChallengeCheck = detectChallenge(finalResult.html, finalResult.statusCode);
+            if (finalChallengeCheck.isChallenge && finalChallengeCheck.confidence >= 0.7) {
+                // Give up — return with warning flag
+                finalResult = { ...finalResult, challengeDetected: true };
+            }
+        }
+    }
+    if (canUseCache && !finalResult.challengeDetected) {
+        hooks.setCache?.(url, finalResult) ?? setBasicCache(url, finalResult);
     }
-    recordMethod(browserResult.method);
-    return browserResult;
+    recordMethod(finalResult.method);
+    return finalResult;
 }
 /* ---------- legacy export for tests ------------------------------------- */
 /**