webpeel 0.7.0 → 0.7.1

package/dist/core/fetcher.d.ts

@@ -1,11 +1,16 @@
 /**
  * Core fetching logic: simple HTTP and browser-based fetching
  */
+import type { PageAction } from '../types.js';
 export interface FetchResult {
+    /** Text content (HTML/JSON/XML/plain text). For binary documents, this may be an empty string. */
     html: string;
+    /** Raw response body (used for binary documents like PDFs/DOCX). */
+    buffer?: Buffer;
     url: string;
     statusCode?: number;
     screenshot?: Buffer;
+    /** Raw Content-Type header from the response (may include charset). */
     contentType?: string;
     /** Playwright page object (only available in browser/stealth mode, must be closed by caller) */
     page?: import('playwright-core').Page;
@@ -17,7 +22,9 @@ export interface FetchResult {
  * Fast and lightweight, but can be blocked by Cloudflare/bot detection
  * SECURITY: Manual redirect handling with SSRF re-validation
  */
-export declare function simpleFetch(url: string, userAgent?: string, timeoutMs?: number, customHeaders?: Record<string, string>): Promise<FetchResult>;
+export declare function simpleFetch(url: string, userAgent?: string, timeoutMs?: number, customHeaders?: Record<string, string>, abortSignal?: AbortSignal): Promise<FetchResult>;
+export declare function closePool(): Promise<void>;
+export declare function warmup(): Promise<void>;
 /**
  * Fetch using headless Chromium via Playwright
  * Slower but can handle JavaScript-heavy sites and bypass some bot detection
@@ -31,6 +38,27 @@ export declare function browserFetch(url: string, options?: {
     headers?: Record<string, string>;
     cookies?: string[];
     stealth?: boolean;
+    actions?: PageAction[];
+    /** Keep the browser page open after fetch (caller must close page + browser) */
+    keepPageOpen?: boolean;
+    /** Abort signal for internal races/cancellation */
+    signal?: AbortSignal;
+}): Promise<FetchResult>;
+/**
+ * Retry a fetch operation with exponential backoff
+ */
+export declare function browserScreenshot(url: string, options?: {
+    fullPage?: boolean;
+    width?: number;
+    height?: number;
+    format?: 'png' | 'jpeg';
+    quality?: number;
+    waitMs?: number;
+    timeoutMs?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
     actions?: Array<{
         type: 'wait' | 'click' | 'scroll' | 'type' | 'fill' | 'select' | 'press' | 'hover' | 'waitForSelector' | 'screenshot';
         selector?: string;
@@ -40,12 +68,10 @@ export declare function browserFetch(url: string, options?: {
         to?: 'top' | 'bottom' | number;
         timeout?: number;
     }>;
-    /** Keep the browser page open after fetch (caller must close page + browser) */
-    keepPageOpen?: boolean;
-}): Promise<FetchResult>;
-/**
- * Retry a fetch operation with exponential backoff
- */
+}): Promise<{
+    buffer: Buffer;
+    finalUrl: string;
+}>;
 export declare function retryFetch<T>(fn: () => Promise<T>, maxAttempts?: number, baseDelayMs?: number): Promise<T>;
 /**
  * Clean up browser resources

package/dist/core/fetcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAiQH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gGAAgG;IAChG,IAAI,CAAC,EAAE,OAAO,iBAAiB,EAAE,IAAI,CAAC;IACtC,mGAAmG;IACnG,OAAO,CAAC,EAAE,OAAO,iBAAiB,EAAE,OAAO,CAAC;CAC7C;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,MAAM,EACX,SAAS,CAAC,EAAE,MAAM,EAClB,SAAS,GAAE,MAAc,EACzB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACrC,OAAO,CAAC,WAAW,CAAC,CAgMtB;AAyCD;;;GAGG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,iBAAiB,GAAG,YAAY,CAAC;QACtH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;QAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,gFAAgF;IAChF,YAAY,CAAC,EAAE,OAAO,CAAC;CACnB,GACL,OAAO,CAAC,WAAW,CAAC,CAyLtB;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,CAAC,EAChC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,WAAW,GAAE,MAAU,EACvB,WAAW,GAAE,MAAa,GACzB,OAAO,CAAC,CAAC,CAAC,CAsBZ;AAED;;GAEG;AACH,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAS7C"}
+{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAcH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAsX9C,MAAM,WAAW,WAAW;IAC1B,kGAAkG;IAClG,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gGAAgG;IAChG,IAAI,CAAC,EAAE,OAAO,iBAAiB,EAAE,IAAI,CAAC;IACtC,mGAAmG;IACnG,OAAO,CAAC,EAAE,OAAO,iBAAiB,EAAE,OAAO,CAAC;CAC7C;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,MAAM,EACX,SAAS,CAAC,EAAE,MAAM,EAClB,SAAS,GAAE,MAAc,EACzB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACtC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,WAAW,CAAC,CA4RtB;AAED,wBAAsB,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAI/C;AA2FD,wBAAsB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAI5C;AA4CD;;;GAGG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IACvB,gFAAgF;IAChF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,mDAAmD;IACnD,MAAM,CAAC,EAAE,WAAW,CAAC;CACjB,GACL,OAAO,CAAC,WAAW,CAAC,CAsUtB;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,iBAAiB,GAAG,YAAY,CAAC;QACtH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;QAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;CACC,GACL,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CA2M/C;AAED,wBAAsB,UAAU,CAAC,CAAC,EAChC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,WAAW,GAAE,MAAU,EACvB,WAAW,GAAE,MAAa,GACzB,OAAO,CAAC,CAAC,CAAC,CAsBZ;AAED;;GAEG;AACH,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAkB7C"}