webpeel 0.21.80 → 0.21.81

package/dist/core/domain-extractors-basic.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Basic domain extraction — public/free tier.
+ *
+ * Handles a few common domains with simple logic.
+ * Full 55+ domain extractors are premium/server-only.
+ *
+ * This module is safe to include in the npm package.
+ * The full `domain-extractors.ts` is compiled for the server
+ * but wired in only when premium hooks are registered.
+ */
+import type { DomainExtractResult } from './domain-extractors.js';
+/**
+ * Basic domain data extractor — free tier stub.
+ *
+ * Always returns null (delegates all extraction to the normal pipeline).
+ * Premium servers override this via the `extractDomainData` strategy hook.
+ */
+export declare function extractDomainDataBasic(_html: string, _url: string): Promise<DomainExtractResult | null>;
+/**
+ * Basic domain extractor lookup — free tier stub.
+ *
+ * Always returns null (no domain is recognized in basic mode).
+ * Premium servers override this via the `getDomainExtractor` strategy hook.
+ */
+export declare function getDomainExtractorBasic(_url: string): ((html: string, url: string) => Promise<DomainExtractResult | null>) | null;

package/dist/core/domain-extractors-basic.js

@@ -0,0 +1,31 @@
+/**
+ * Basic domain extraction — public/free tier.
+ *
+ * Handles a few common domains with simple logic.
+ * Full 55+ domain extractors are premium/server-only.
+ *
+ * This module is safe to include in the npm package.
+ * The full `domain-extractors.ts` is compiled for the server
+ * but wired in only when premium hooks are registered.
+ */
+/**
+ * Basic domain data extractor — free tier stub.
+ *
+ * Always returns null (delegates all extraction to the normal pipeline).
+ * Premium servers override this via the `extractDomainData` strategy hook.
+ */
+export async function extractDomainDataBasic(_html, _url) {
+    // Basic (free) tier: no domain-specific extraction.
+    // The normal fetch + markdown pipeline handles everything.
+    // Premium hook provides 55+ domain extractors (Twitter, Reddit, GitHub, HN, etc.)
+    return null;
+}
+/**
+ * Basic domain extractor lookup — free tier stub.
+ *
+ * Always returns null (no domain is recognized in basic mode).
+ * Premium servers override this via the `getDomainExtractor` strategy hook.
+ */
+export function getDomainExtractorBasic(_url) {
+    return null;
+}

package/dist/core/pipeline.js

@@ -14,7 +14,28 @@ import { autoScroll as runAutoScroll } from './actions.js';
 import { extractStructured } from './extract.js';
 import { isPdfContentType, isDocxContentType, extractDocumentToFormat } from './documents.js';
 import { parseYouTubeUrl, getYouTubeTranscript } from './youtube.js';
-import { extractDomainData, getDomainExtractor } from './domain-extractors.js';
+import { extractDomainDataBasic, getDomainExtractorBasic } from './domain-extractors-basic.js';
+import { getDomainExtractHook, getDomainExtractorHook, getSPADomainsHook, getSPAPatternsHook } from './strategy-hooks.js';
+// Lazy-loaded full extractors — available in repo/server, absent in npm package.
+// The dynamic import avoids hard failures when domain-extractors.js is excluded from npm.
+let _fullExtractorsLoaded = false;
+let _fullExtractDomainData = null;
+let _fullGetDomainExtractor = null;
+async function loadFullExtractors() {
+    if (_fullExtractorsLoaded)
+        return;
+    _fullExtractorsLoaded = true;
+    try {
+        const mod = await import('./domain-extractors.js');
+        _fullExtractDomainData = mod.extractDomainData;
+        _fullGetDomainExtractor = mod.getDomainExtractor;
+    }
+    catch {
+        // Not available (npm package) — basic stubs will be used
+    }
+}
+// Eagerly start loading (non-blocking)
+loadFullExtractors();
 import { extractReadableContent } from './readability.js';
 import { quickAnswer as runQuickAnswer } from './quick-answer.js';
 import { Timer } from './timing.js';
@@ -24,6 +45,38 @@ import { sanitizeForLLM } from './prompt-guard.js';
 import { getSourceCredibility } from './source-credibility.js';
 import { createLogger } from './logger.js';
 const log = createLogger('pipeline');
+// ---------------------------------------------------------------------------
+// Hook-aware wrappers — route through premium hooks, fall back to basic stubs
+// ---------------------------------------------------------------------------
+/**
+ * Check if a URL has a domain extractor.
+ * Priority: premium hook → full extractors (repo/server) → basic stub.
+ */
+function hasDomainExtractor(url) {
+    const hookFn = getDomainExtractorHook();
+    if (hookFn)
+        return hookFn(url) !== null;
+    // Full extractors available (repo/server build)?
+    if (_fullGetDomainExtractor)
+        return _fullGetDomainExtractor(url) !== null;
+    // npm package fallback — basic stubs
+    return getDomainExtractorBasic(url) !== null;
+}
+/**
+ * Run domain extraction on HTML/URL.
+ * Priority: premium hook → full extractors (repo/server) → basic stub.
+ */
+async function runDomainExtract(html, url) {
+    const hookFn = getDomainExtractHook();
+    if (hookFn)
+        return hookFn(html, url);
+    // Full extractors available (repo/server build)?
+    await loadFullExtractors(); // Ensure loaded
+    if (_fullExtractDomainData)
+        return _fullExtractDomainData(html, url);
+    // npm package fallback — basic stubs
+    return extractDomainDataBasic(html, url);
+}
 /** Create the initial PipelineContext with defaults */
 export function createContext(url, options) {
     return {
@@ -147,27 +200,16 @@ export function normalizeOptions(ctx) {
         ctx.render = true;
     }
     // Auto-detect SPAs that require browser rendering (no --render flag needed)
+    // Premium hook provides full SPA domain list; basic has a small default set.
     if (!ctx.render) {
-        const SPA_DOMAINS = new Set([
-            'www.google.com', // Google Flights, Maps, Shopping etc.
-            'flights.google.com',
-            'www.airbnb.com',
-            'www.booking.com',
-            'www.expedia.com',
-            'www.kayak.com',
-            'www.skyscanner.com',
-            'www.tripadvisor.com',
-            'www.indeed.com',
-            'www.glassdoor.com',
-            'www.zillow.com', // already handled but backup
-            'app.webpeel.dev', // our own dashboard is a SPA
-        ]);
-        // More specific: some google.com paths need render, not all
-        const SPA_URL_PATTERNS = [
-            /google\.com\/travel/,
-            /google\.com\/maps/,
-            /google\.com\/shopping/,
-        ];
+        const spaDomainsHook = getSPADomainsHook();
+        const spaPatternsHook = getSPAPatternsHook();
+        // Basic SPA defaults — minimal set for free tier
+        const DEFAULT_SPA_DOMAINS = new Set([]);
+        const DEFAULT_SPA_PATTERNS = [];
+        // Premium hook merges its full list; basic uses defaults
+        const SPA_DOMAINS = spaDomainsHook ? spaDomainsHook() : DEFAULT_SPA_DOMAINS;
+        const SPA_URL_PATTERNS = spaPatternsHook ? spaPatternsHook() : DEFAULT_SPA_PATTERNS;
         try {
             const hostname = new URL(ctx.url).hostname;
             if (SPA_DOMAINS.has(hostname)) {
@@ -304,10 +346,10 @@ export async function fetchContent(ctx) {
     const needsDesignAnalysis = ctx.options.designAnalysis && ctx.render;
     // Try API-based domain extraction first (Reddit, GitHub, HN use APIs, not HTML)
     // This avoids expensive browser fetches that often get blocked
-    if (getDomainExtractor(ctx.url)) {
+    if (hasDomainExtractor(ctx.url)) {
         try {
             ctx.timer.mark('domainApiFirst');
-            const ddResult = await extractDomainData('', ctx.url);
+            const ddResult = await runDomainExtract('', ctx.url);
             ctx.timer.end('domainApiFirst');
             if (ddResult && ddResult.cleanContent.length > 50) {
                 ctx.domainData = ddResult;
@@ -385,9 +427,9 @@ export async function fetchContent(ctx) {
     }
     catch (fetchError) {
         // If fetch failed but we have a domain extractor, try it as fallback
-        if (getDomainExtractor(ctx.url)) {
+        if (hasDomainExtractor(ctx.url)) {
             try {
-                const ddResult = await extractDomainData('', ctx.url);
+                const ddResult = await runDomainExtract('', ctx.url);
                 if (ddResult && ddResult.cleanContent.length > 50) {
                     ctx.timer.end('fetch');
                     ctx.domainData = ddResult;
@@ -1041,14 +1083,14 @@ export async function postProcess(ctx) {
     }
     // Domain-aware structured extraction (Twitter, Reddit, GitHub, HN)
     // Fires when URL matches a known domain. Replaces content with clean markdown.
-    if (getDomainExtractor(fetchResult.url) && !ctx.domainApiHandled) {
+    if (hasDomainExtractor(fetchResult.url) && !ctx.domainApiHandled) {
         try {
             ctx.timer.mark('domainExtract');
             // Try raw HTML first, then fall back to readability-processed content
             // (some SPAs like Google Flights have data only after readability processing)
-            let ddResult = await extractDomainData(fetchResult.html, fetchResult.url);
+            let ddResult = await runDomainExtract(fetchResult.html, fetchResult.url);
             if (!ddResult && ctx.content) {
-                ddResult = await extractDomainData(ctx.content, fetchResult.url);
+                ddResult = await runDomainExtract(ctx.content, fetchResult.url);
             }
             ctx.timer.end('domainExtract');
             if (ddResult) {

package/dist/core/strategy-hooks.d.ts

@@ -10,6 +10,7 @@
  * All hook methods are optional — unset hooks are simply skipped.
  */
 import type { FetchResult } from './fetcher.js';
+import type { DomainExtractResult } from './domain-extractors.js';
 export interface StrategyResult extends FetchResult {
     method: 'simple' | 'browser' | 'stealth' | 'cached' | 'cloaked' | 'cycle' | 'peeltls' | 'cf-worker' | 'google-cache';
     /**
@@ -65,6 +66,39 @@ export interface StrategyHooks {
      * Only called when `shouldRace()` returns true.  Default: 2000.
      */
     getRaceTimeoutMs?(): number;
+    /**
+     * Premium domain extraction hook — 55+ domain extractors.
+     * Return null to fall back to basic/no extraction.
+     */
+    extractDomainData?(html: string, url: string): Promise<DomainExtractResult | null>;
+    /**
+     * Returns a function that checks if a URL has a known domain extractor.
+     * Premium knows which domains have extractors; basic returns null for all.
+     */
+    getDomainExtractor?(url: string): ((html: string, url: string) => Promise<DomainExtractResult | null>) | null;
+    /**
+     * Premium SPA domain list — knows which sites require browser rendering.
+     * Basic: returns empty set (no SPA auto-detection).
+     */
+    getSPADomains?(): Set<string>;
+    /**
+     * Premium SPA URL patterns — matches specific paths needing render.
+     * Basic: returns empty array.
+     */
+    getSPAPatterns?(): RegExp[];
+    /**
+     * Premium CAPTCHA/challenge solving hook.
+     * Return null to fall back to default challenge handling.
+     */
+    solveChallenge?(page: any, url: string): Promise<{
+        solved: boolean;
+        html?: string;
+    } | null>;
+    /**
+     * Premium wait-for-stable content logic — smarter than waitForLoadState.
+     * Return null/undefined to fall back to default wait logic.
+     */
+    waitForContentStable?(page: any, options?: any): Promise<void>;
 }
 /**
  * Register premium strategy hooks.  Should be called once at server startup.
@@ -79,3 +113,33 @@ export declare function clearStrategyHooks(): void;
  * Retrieve the current hooks (internal — used by strategies.ts).
  */
 export declare function getStrategyHooks(): Readonly<StrategyHooks>;
+/**
+ * Get the premium domain extraction hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getDomainExtractHook(): StrategyHooks['extractDomainData'];
+/**
+ * Get the premium domain extractor lookup hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getDomainExtractorHook(): StrategyHooks['getDomainExtractor'];
+/**
+ * Get the premium SPA domains hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getSPADomainsHook(): StrategyHooks['getSPADomains'];
+/**
+ * Get the premium SPA patterns hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getSPAPatternsHook(): StrategyHooks['getSPAPatterns'];
+/**
+ * Get the premium challenge solver hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getChallengeHook(): StrategyHooks['solveChallenge'];
+/**
+ * Get the premium content stability hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getStabilityHook(): StrategyHooks['waitForContentStable'];

package/dist/core/strategy-hooks.js

@@ -30,3 +30,45 @@ export function clearStrategyHooks() {
 export function getStrategyHooks() {
     return registeredHooks;
 }
+/**
+ * Get the premium domain extraction hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getDomainExtractHook() {
+    return registeredHooks.extractDomainData;
+}
+/**
+ * Get the premium domain extractor lookup hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getDomainExtractorHook() {
+    return registeredHooks.getDomainExtractor;
+}
+/**
+ * Get the premium SPA domains hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getSPADomainsHook() {
+    return registeredHooks.getSPADomains;
+}
+/**
+ * Get the premium SPA patterns hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getSPAPatternsHook() {
+    return registeredHooks.getSPAPatterns;
+}
+/**
+ * Get the premium challenge solver hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getChallengeHook() {
+    return registeredHooks.solveChallenge;
+}
+/**
+ * Get the premium content stability hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getStabilityHook() {
+    return registeredHooks.waitForContentStable;
+}

package/dist/server/premium/challenge.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Premium challenge solver — server-only wrapper.
+ *
+ * Re-exports the challenge-solver functionality for use as a strategy hook.
+ * The npm package handles challenges inline in pipeline.ts (basic handling).
+ * Premium servers can wire in enhanced challenge solving via hooks.
+ */
+export { solveChallenge } from '../../core/challenge-solver.js';

package/dist/server/premium/challenge.js

@@ -0,0 +1,8 @@
+/**
+ * Premium challenge solver — server-only wrapper.
+ *
+ * Re-exports the challenge-solver functionality for use as a strategy hook.
+ * The npm package handles challenges inline in pipeline.ts (basic handling).
+ * Premium servers can wire in enhanced challenge solving via hooks.
+ */
+export { solveChallenge } from '../../core/challenge-solver.js';

package/dist/server/premium/extractors.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Premium domain extractors — server-only wrapper.
+ *
+ * Re-exports the full extractDomainData and getDomainExtractor functions
+ * from core/domain-extractors.ts for use as strategy hooks.
+ *
+ * The npm package uses basic stubs (always return null).
+ * When premium hooks are registered, these full extractors are wired in.
+ */
+export { extractDomainData, getDomainExtractor } from '../../core/domain-extractors.js';

package/dist/server/premium/extractors.js

@@ -0,0 +1,10 @@
+/**
+ * Premium domain extractors — server-only wrapper.
+ *
+ * Re-exports the full extractDomainData and getDomainExtractor functions
+ * from core/domain-extractors.ts for use as strategy hooks.
+ *
+ * The npm package uses basic stubs (always return null).
+ * When premium hooks are registered, these full extractors are wired in.
+ */
+export { extractDomainData, getDomainExtractor } from '../../core/domain-extractors.js';

package/dist/server/premium/index.d.ts

@@ -5,6 +5,9 @@
  *   • SWR (stale-while-revalidate) response cache
  *   • Domain intelligence (learns which sites need browser/stealth)
  *   • Parallel race strategy (starts browser if simple fetch is slow)
+ *   • 55+ domain extractors (Twitter, Reddit, GitHub, HN, Wikipedia, etc.)
+ *   • SPA auto-detection (travel, jobs, real estate sites)
+ *   • Content stability detection (smart DOM mutation monitoring)
  *
  * These modules are NOT shipped in the npm package.
  */

package/dist/server/premium/index.js

@@ -5,12 +5,18 @@
  *   • SWR (stale-while-revalidate) response cache
  *   • Domain intelligence (learns which sites need browser/stealth)
  *   • Parallel race strategy (starts browser if simple fetch is slow)
+ *   • 55+ domain extractors (Twitter, Reddit, GitHub, HN, Wikipedia, etc.)
+ *   • SPA auto-detection (travel, jobs, real estate sites)
+ *   • Content stability detection (smart DOM mutation monitoring)
  *
  * These modules are NOT shipped in the npm package.
  */
 import { registerStrategyHooks } from '../../core/strategy-hooks.js';
 import { createSWRCacheHooks } from './swr-cache.js';
 import { createDomainIntelHooks } from './domain-intel.js';
+import { extractDomainData, getDomainExtractor } from './extractors.js';
+import { SPA_DOMAINS, SPA_URL_PATTERNS } from './spa-detection.js';
+import { waitForContentStable } from './stability.js';
 export { clearDomainIntel } from './domain-intel.js';
 /**
  * Wire all premium hooks into the core strategy layer.
@@ -31,5 +37,14 @@ export function registerPremiumHooks() {
         // Parallel race strategy
         shouldRace: () => true,
         getRaceTimeoutMs: () => 2000,
+        // Premium domain extraction (55+ extractors)
+        extractDomainData,
+        // Premium domain extractor lookup
+        getDomainExtractor: (url) => getDomainExtractor(url),
+        // Premium SPA detection
+        getSPADomains: () => SPA_DOMAINS,
+        getSPAPatterns: () => SPA_URL_PATTERNS,
+        // Premium content stability (DOM mutation monitoring)
+        waitForContentStable,
     });
 }

package/dist/server/premium/spa-detection.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Premium SPA detection — server-only.
+ *
+ * Full list of domains and URL patterns that require browser rendering.
+ * The npm package only has a minimal default set (Google, our own dashboard).
+ * Premium servers register these via strategy hooks.
+ */
+/**
+ * Domains that are known SPAs requiring browser rendering.
+ * Includes travel, real estate, job boards, and other dynamic sites.
+ */
+export declare const SPA_DOMAINS: Set<string>;
+/**
+ * URL patterns that match SPA routes on mixed-content domains.
+ * E.g. google.com/travel is SPA, but google.com/search is not.
+ */
+export declare const SPA_URL_PATTERNS: RegExp[];

package/dist/server/premium/spa-detection.js

@@ -0,0 +1,39 @@
+/**
+ * Premium SPA detection — server-only.
+ *
+ * Full list of domains and URL patterns that require browser rendering.
+ * The npm package only has a minimal default set (Google, our own dashboard).
+ * Premium servers register these via strategy hooks.
+ */
+/**
+ * Domains that are known SPAs requiring browser rendering.
+ * Includes travel, real estate, job boards, and other dynamic sites.
+ */
+export const SPA_DOMAINS = new Set([
+    // Google properties
+    'www.google.com',
+    'flights.google.com',
+    // Travel
+    'www.airbnb.com',
+    'www.booking.com',
+    'www.expedia.com',
+    'www.kayak.com',
+    'www.skyscanner.com',
+    'www.tripadvisor.com',
+    // Jobs
+    'www.indeed.com',
+    'www.glassdoor.com',
+    // Real estate
+    'www.zillow.com',
+    // Our own dashboard
+    'app.webpeel.dev',
+]);
+/**
+ * URL patterns that match SPA routes on mixed-content domains.
+ * E.g. google.com/travel is SPA, but google.com/search is not.
+ */
+export const SPA_URL_PATTERNS = [
+    /google\.com\/travel/,
+    /google\.com\/maps/,
+    /google\.com\/shopping/,
+];

package/dist/server/premium/stability.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Premium content stability detection — server-only.
+ *
+ * Provides smarter content-stability waiting logic than the default
+ * waitForLoadState('networkidle'). Monitors DOM mutations and network
+ * activity to determine when a page has truly finished rendering.
+ *
+ * The npm package uses default Playwright waitForLoadState.
+ * Premium servers can wire this in via the waitForContentStable hook.
+ */
+export interface StabilityOptions {
+    /** Maximum time to wait (ms). Default: 5000. */
+    timeoutMs?: number;
+    /** Minimum quiet period before declaring stable (ms). Default: 500. */
+    quietMs?: number;
+}
+/**
+ * Wait for page content to stabilize by monitoring DOM mutations.
+ *
+ * More reliable than waitForLoadState('networkidle') for SPAs that
+ * progressively render content.
+ */
+export declare function waitForContentStable(page: any, options?: StabilityOptions): Promise<void>;

package/dist/server/premium/stability.js

@@ -0,0 +1,58 @@
+/**
+ * Premium content stability detection — server-only.
+ *
+ * Provides smarter content-stability waiting logic than the default
+ * waitForLoadState('networkidle'). Monitors DOM mutations and network
+ * activity to determine when a page has truly finished rendering.
+ *
+ * The npm package uses default Playwright waitForLoadState.
+ * Premium servers can wire this in via the waitForContentStable hook.
+ */
+/**
+ * Wait for page content to stabilize by monitoring DOM mutations.
+ *
+ * More reliable than waitForLoadState('networkidle') for SPAs that
+ * progressively render content.
+ */
+export async function waitForContentStable(page, options) {
+    const timeout = options?.timeoutMs ?? 5000;
+    const quiet = options?.quietMs ?? 500;
+    const start = Date.now();
+    // Use page.evaluate to monitor DOM mutations
+    await page.evaluate(({ quietMs, timeoutMs }) => {
+        return new Promise((resolve) => {
+            let lastMutation = Date.now();
+            let settled = false;
+            const observer = new MutationObserver(() => {
+                lastMutation = Date.now();
+            });
+            observer.observe(document.body, {
+                childList: true,
+                subtree: true,
+                characterData: true,
+            });
+            const check = () => {
+                const now = Date.now();
+                if (now - lastMutation >= quietMs || settled) {
+                    observer.disconnect();
+                    resolve();
+                    return;
+                }
+                if (now - lastMutation > timeoutMs) {
+                    observer.disconnect();
+                    resolve();
+                    return;
+                }
+                requestAnimationFrame(check);
+            };
+            // Hard timeout
+            setTimeout(() => {
+                settled = true;
+                observer.disconnect();
+                resolve();
+            }, timeoutMs);
+            // Start checking after an initial quiet period
+            setTimeout(check, quietMs);
+        });
+    }, { quietMs: quiet, timeoutMs: Math.max(0, timeout - (Date.now() - start)) });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.80",
+  "version": "0.21.81",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",