webpeel 0.16.0 → 0.17.0

package/dist/core/peel-tls.d.ts

@@ -0,0 +1,26 @@
+/**
+ * PeelTLS — WebPeel's TLS Fingerprint Spoofing Engine
+ * BSD-licensed replacement for CycleTLS (GPL-3).
+ *
+ * Manages a singleton Go binary process that provides TLS fingerprint spoofing
+ * via uTLS (utls library). The binary exposes a local HTTP API that this module
+ * communicates with.
+ */
+import type { FetchResult } from './fetcher.js';
+export interface PeelTLSOptions {
+    timeout?: number;
+    proxy?: string;
+    headers?: Record<string, string>;
+    /** TLS fingerprint preset: 'chrome-133' | 'firefox-120' | 'safari-16' | raw JA3 string */
+    fingerprint?: string;
+}
+export interface PeelTLSResult extends FetchResult {
+    method: 'peeltls';
+}
+/** Check if the PeelTLS binary is available on this platform. */
+export declare function isPeelTLSAvailable(): boolean;
+/** Fetch a URL using PeelTLS TLS fingerprint spoofing. */
+export declare function peelTLSFetch(url: string, options?: PeelTLSOptions): Promise<PeelTLSResult>;
+/** Gracefully shut down the PeelTLS binary process. */
+export declare function shutdownPeelTLS(): Promise<void>;
+//# sourceMappingURL=peel-tls.d.ts.map

package/dist/core/peel-tls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"peel-tls.d.ts","sourceRoot":"","sources":["../../src/core/peel-tls.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AASH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAiBhD,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,0FAA0F;IAC1F,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,aAAc,SAAQ,WAAW;IAChD,MAAM,EAAE,SAAS,CAAC;CACnB;AAuDD,iEAAiE;AACjE,wBAAgB,kBAAkB,IAAI,OAAO,CAO5C;AA+ED,0DAA0D;AAC1D,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAsDhG;AAED,uDAAuD;AACvD,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAiBrD"}

package/dist/core/peel-tls.js

@@ -0,0 +1,221 @@
+/**
+ * PeelTLS — WebPeel's TLS Fingerprint Spoofing Engine
+ * BSD-licensed replacement for CycleTLS (GPL-3).
+ *
+ * Manages a singleton Go binary process that provides TLS fingerprint spoofing
+ * via uTLS (utls library). The binary exposes a local HTTP API that this module
+ * communicates with.
+ */
+import { existsSync } from 'fs';
+import { resolve as pathResolve } from 'path';
+import { spawn } from 'child_process';
+import { randomBytes } from 'crypto';
+import { platform, arch } from 'os';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+// Chrome 133 user agent (matches Go binary's default fingerprint)
+const CHROME_133_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36';
+const DEFAULT_HEADERS = {
+    'Sec-Ch-Ua': '"Chromium";v="133", "Google Chrome";v="133"',
+    'Sec-Ch-Ua-Mobile': '?0',
+    'Sec-Ch-Ua-Platform': '"macOS"',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9',
+};
+// ─── Singleton state ──────────────────────────────────────────────────────────
+let binaryProcess = null;
+let binaryPort = null;
+let binaryToken = null;
+let startPromise = null;
+// ─── Binary discovery ─────────────────────────────────────────────────────────
+/** Get the binary path for the current platform/arch. */
+function getBinaryPath() {
+    const plat = platform(); // 'darwin' | 'linux' | 'win32'
+    const ar = arch(); // 'x64' | 'arm64'
+    const platMap = {
+        darwin: 'darwin',
+        linux: 'linux',
+        win32: 'windows',
+    };
+    const archMap = {
+        x64: 'x64',
+        arm64: 'arm64',
+    };
+    const suffix = plat === 'win32' ? '.exe' : '';
+    const name = `peeltls-${platMap[plat] ?? plat}-${archMap[ar] ?? ar}${suffix}`;
+    // Resolve __dirname for both ESM and CJS
+    let thisDir;
+    try {
+        thisDir = dirname(fileURLToPath(import.meta.url));
+    }
+    catch {
+        thisDir = __dirname ?? process.cwd();
+    }
+    const candidates = [
+        // Relative to this file: src/core/ -> ../../peeltls/dist/
+        pathResolve(thisDir, '..', '..', 'peeltls', 'dist', name),
+        // From project root (cwd)
+        pathResolve(process.cwd(), 'peeltls', 'dist', name),
+        // Fallback: just 'peeltls' in dist (for development)
+        pathResolve(process.cwd(), 'peeltls', 'dist', 'peeltls'),
+    ];
+    for (const candidate of candidates) {
+        if (existsSync(candidate))
+            return candidate;
+    }
+    throw new Error(`PeelTLS binary not found for ${plat}/${ar}.\nExpected: ${candidates[0]}\nBuild it with: cd peeltls && bash build.sh`);
+}
+/** Check if the PeelTLS binary is available on this platform. */
+export function isPeelTLSAvailable() {
+    try {
+        getBinaryPath();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ─── Process management ───────────────────────────────────────────────────────
+/** Start the PeelTLS Go binary (singleton — only spawned once). */
+async function ensureRunning() {
+    // Already running
+    if (binaryPort !== null && binaryToken !== null && binaryProcess !== null && !binaryProcess.killed) {
+        return { port: binaryPort, token: binaryToken };
+    }
+    // Another call already started it
+    if (startPromise !== null) {
+        await startPromise;
+        return { port: binaryPort, token: binaryToken };
+    }
+    startPromise = new Promise((resolve, reject) => {
+        const binPath = getBinaryPath();
+        const token = randomBytes(32).toString('hex');
+        const proc = spawn(binPath, ['--port', '0', '--token', token], {
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        let stdoutBuf = '';
+        const startupTimeout = setTimeout(() => {
+            proc.kill();
+            reject(new Error('PeelTLS binary failed to start within 10s'));
+        }, 10_000);
+        proc.stdout.on('data', (chunk) => {
+            stdoutBuf += chunk.toString();
+            const newlineIdx = stdoutBuf.indexOf('\n');
+            if (newlineIdx !== -1) {
+                clearTimeout(startupTimeout);
+                try {
+                    const info = JSON.parse(stdoutBuf.slice(0, newlineIdx));
+                    binaryPort = info.port;
+                    binaryToken = token;
+                    binaryProcess = proc;
+                    resolve();
+                }
+                catch {
+                    proc.kill();
+                    reject(new Error(`PeelTLS binary output invalid JSON: ${stdoutBuf}`));
+                }
+            }
+        });
+        proc.stderr.on('data', (chunk) => {
+            if (process.env['DEBUG']) {
+                process.stderr.write(`[peeltls] ${chunk.toString().trim()}\n`);
+            }
+        });
+        proc.on('error', (err) => {
+            clearTimeout(startupTimeout);
+            reject(new Error(`PeelTLS binary failed to spawn: ${err.message}`));
+        });
+        proc.on('exit', (code) => {
+            binaryProcess = null;
+            binaryPort = null;
+            binaryToken = null;
+            startPromise = null;
+            if (code !== 0 && code !== null) {
+                clearTimeout(startupTimeout);
+                // Don't reject here if already resolved — just clean state
+            }
+        });
+    });
+    await startPromise;
+    startPromise = null; // Reset so it can restart if the process exits later
+    return { port: binaryPort, token: binaryToken };
+}
+// ─── Public API ───────────────────────────────────────────────────────────────
+/** Fetch a URL using PeelTLS TLS fingerprint spoofing. */
+export async function peelTLSFetch(url, options) {
+    const { port, token } = await ensureRunning();
+    const mergedHeaders = {
+        'User-Agent': CHROME_133_UA,
+        ...DEFAULT_HEADERS,
+        ...(options?.headers ?? {}),
+    };
+    const requestBody = JSON.stringify({
+        url,
+        method: 'GET',
+        headers: mergedHeaders,
+        fingerprint: options?.fingerprint ?? 'chrome-133',
+        proxy: options?.proxy ?? '',
+        timeout: Math.round((options?.timeout ?? 30_000) / 1000),
+        followRedirects: true,
+        maxRedirects: 10,
+    });
+    if (process.env['DEBUG']) {
+        process.stderr.write(`[peeltls] fetch: ${url}\n`);
+    }
+    const timeoutMs = options?.timeout ?? 30_000;
+    const response = await fetch(`http://127.0.0.1:${port}/fetch`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${token}`,
+        },
+        body: requestBody,
+        signal: AbortSignal.timeout(timeoutMs + 5_000), // extra buffer over Go's own timeout
+    });
+    const result = (await response.json());
+    if (result.error) {
+        throw new Error(`PeelTLS fetch failed: ${result.error}`);
+    }
+    return {
+        html: result.body ?? '',
+        url: result.finalUrl ?? url,
+        statusCode: result.status ?? 0,
+        contentType: result.headers?.['content-type'] ?? 'text/html',
+        method: 'peeltls',
+    };
+}
+/** Gracefully shut down the PeelTLS binary process. */
+export async function shutdownPeelTLS() {
+    if (binaryPort === null || binaryToken === null || binaryProcess === null)
+        return;
+    try {
+        await fetch(`http://127.0.0.1:${binaryPort}/shutdown`, {
+            method: 'POST',
+            headers: { Authorization: `Bearer ${binaryToken}` },
+            signal: AbortSignal.timeout(5_000),
+        });
+    }
+    catch {
+        // Force kill if graceful shutdown fails or times out
+        binaryProcess?.kill('SIGTERM');
+    }
+    binaryProcess = null;
+    binaryPort = null;
+    binaryToken = null;
+}
+// ─── Process exit cleanup ─────────────────────────────────────────────────────
+process.on('exit', () => {
+    binaryProcess?.kill('SIGTERM');
+});
+process.on('SIGINT', () => {
+    binaryProcess?.kill('SIGTERM');
+    process.exit(0);
+});
+process.on('SIGTERM', () => {
+    binaryProcess?.kill('SIGTERM');
+    process.exit(0);
+});
+//# sourceMappingURL=peel-tls.js.map

package/dist/core/peel-tls.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"peel-tls.js","sourceRoot":"","sources":["../../src/core/peel-tls.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,MAAM,CAAC;AAC9C,OAAO,EAAE,KAAK,EAAqB,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAG/B,kEAAkE;AAClE,MAAM,aAAa,GACjB,uHAAuH,CAAC;AAE1H,MAAM,eAAe,GAA2B;IAC9C,WAAW,EAAE,6CAA6C;IAC1D,kBAAkB,EAAE,IAAI;IACxB,oBAAoB,EAAE,SAAS;IAC/B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,MAAM;IACxB,MAAM,EAAE,iEAAiE;IACzE,iBAAiB,EAAE,gBAAgB;CACpC,CAAC;AAcF,iFAAiF;AAEjF,IAAI,aAAa,GAAwB,IAAI,CAAC;AAC9C,IAAI,UAAU,GAAkB,IAAI,CAAC;AACrC,IAAI,WAAW,GAAkB,IAAI,CAAC;AACtC,IAAI,YAAY,GAAyB,IAAI,CAAC;AAE9C,iFAAiF;AAEjF,yDAAyD;AACzD,SAAS,aAAa;IACpB,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAC,+BAA+B;IACxD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,kBAAkB;IAErC,MAAM,OAAO,GAA2B;QACtC,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,SAAS;KACjB,CAAC;IACF,MAAM,OAAO,GAA2B;QACtC,GAAG,EAAE,KAAK;QACV,KAAK,EAAE,OAAO;KACf,CAAC;IAEF,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9C,MAAM,IAAI,GAAG,WAAW,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;IAE9E,yCAAyC;IACzC,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,SAAS,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACvC,CAAC;IAED,MAAM,UAAU,GAAG;QACjB,0DAA0D;QAC1D,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC;QACzD,0BAA0B;QAC1B,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC;QACnD,qDAAqD;QACrD,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC;KACzD,CAAC;IAEF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IAC9C,CAAC;IAED,MAAM,IAAI,KAAK,CACb,gCAAgC,IAAI,IAAI,EAAE,gBAAgB,UAAU,CAAC,CAAC,CAAC,8CAA8C,CACtH,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACH,aAAa,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,iFAAiF;AAEjF,mEAAmE;AACnE,KAAK,UAAU,aAAa;IAC1B,kBAAkB;IAClB,IAAI,UAAU,KAAK,IAAI,IAAI,WAAW,KAAK,IAAI,IAAI,aAAa,KAAK,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;QACnG,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;IAClD,CAAC;IAED,kCAAkC;IAClC,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;QAC1B,MAAM,YAAY,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,UAAW,EAAE,KAAK,EAAE,WAAY,EAAE,CAAC;IACpD,CAAC;IAED,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnD,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAE9C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE;YAC7D,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,MAAM,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE;YACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC,CAAC;QACjE,CAAC,EAAE,MAAM,CAAC,CAAC;QAEX,IAAI,CAAC,MAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACxC,SAAS,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3C,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBACtB,YAAY,CAAC,cAAc,CAAC,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAoC,CAAC;oBAC3F,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;oBACvB,WAAW,GAAG,KAAK,CAAC;oBACpB,aAAa,GAAG,IAAI,CAAC;oBACrB,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAAC,MAAM,CAAC;oBACP,IAAI,CAAC,IAAI,EAAE,CAAC;oBACZ,MAAM,CAAC,IAAI,KAAK,CAAC,uCAAuC,SAAS,EAAE,CAAC,CAAC,CAAC;gBACxE,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACxC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACjE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACvB,YAAY,CAAC,cAAc,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACvB,aAAa,GAAG,IAAI,CAAC;YACrB,UAAU,GAAG,IAAI,CAAC;YAClB,WAAW,GAAG,IAAI,CAAC;YACnB,YAAY,GAAG,IAAI,CAAC;YACpB,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAChC,YAAY,CAAC,cAAc,CAAC,CAAC;gBAC7B,2DAA2D;YAC7D,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,CAAC;IACnB,YAAY,GAAG,IAAI,CAAC,CAAC,qDAAqD;IAC1E,OAAO,EAAE,IAAI,EAAE,UAAW,EAAE,KAAK,EAAE,WAAY,EAAE,CAAC;AACpD,CAAC;AAED,iFAAiF;AAEjF,0DAA0D;AAC1D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,OAAwB;IACtE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,aAAa,EAAE,CAAC;IAE9C,MAAM,aAAa,GAA2B;QAC5C,YAAY,EAAE,aAAa;QAC3B,GAAG,eAAe;QAClB,GAAG,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;KAC5B,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,GAAG;QACH,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,aAAa;QACtB,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,YAAY;QACjD,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE;QAC3B,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,IAAI,CAAC;QACxD,eAAe,EAAE,IAAI;QACrB,YAAY,EAAE,EAAE;KACjB,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC;IAC7C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,oBAAoB,IAAI,QAAQ,EAAE;QAC7D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,aAAa,EAAE,UAAU,KAAK,EAAE;SACjC;QACD,IAAI,EAAE,WAAW;QACjB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC,EAAE,qCAAqC;KACtF,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAMpC,CAAC;IAEF,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE;QACvB,GAAG,EAAE,MAAM,CAAC,QAAQ,IAAI,GAAG;QAC3B,UAAU,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC;QAC9B,WAAW,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,cAAc,CAAC,IAAI,WAAW;QAC5D,MAAM,EAAE,SAAS;KAClB,CAAC;AACJ,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,IAAI,UAAU,KAAK,IAAI,IAAI,WAAW,KAAK,IAAI,IAAI,aAAa,KAAK,IAAI;QAAE,OAAO;IAElF,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,oBAAoB,UAAU,WAAW,EAAE;YACrD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,WAAW,EAAE,EAAE;YACnD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;SACnC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,qDAAqD;QACrD,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAED,aAAa,GAAG,IAAI,CAAC;IACrB,UAAU,GAAG,IAAI,CAAC;IAClB,WAAW,GAAG,IAAI,CAAC;AACrB,CAAC;AAED,iFAAiF;AAEjF,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;IACtB,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;AACjC,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;IACxB,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;IACzB,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/core/pipeline.d.ts

@@ -21,7 +21,7 @@ export interface PipelineContext {
     render: boolean;
     stealth: boolean;
     wait: number;
-    format: 'markdown' | 'text' | 'html';
+    format: 'markdown' | 'text' | 'html' | 'clean';
     timeout: number;
     userAgent?: string;
     screenshot: boolean;
@@ -69,6 +69,10 @@ export interface PipelineContext {
     changeResult?: ChangeResult;
     summaryText?: string;
     screenshotBase64?: string;
+    /** True when domain API extraction handled the content (skip redundant extraction) */
+    domainApiHandled?: boolean;
+    /** Non-fatal warnings accumulated during the pipeline run */
+    warnings: string[];
 }
 /** Create the initial PipelineContext with defaults */
 export declare function createContext(url: string, options: PeelOptions): PipelineContext;

package/dist/core/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/core/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAiBH,OAAO,EAA+B,KAAK,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAInF,OAAO,EAAyC,KAAK,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AACzG,OAAO,EAA0B,KAAK,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,EAAiC,KAAK,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC1F,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEzD,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,EAAE,KAAK,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAGlB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,GAAG,EAAE,OAAO,CAAC;IACb,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,CAAC,EAAE,GAAG,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,iBAAiB,CAAC;IAGnC,WAAW,CAAC,EAAE,GAAG,CAAC;IAGlB,WAAW,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;IAG3D,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,GAAG,CAAC;IACd,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,SAAS,EAAE,MAAM,CAAC;IAGlB,SAAS,CAAC,EAAE;QACV,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IAGF,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,UAAU,CAAC,EAAE,mBAAmB,CAAC;IACjC,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,uDAAuD;AACvD,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,eAAe,CA+ChF;AAMD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CA+F3D;AAMD;;;GAGG;AACH,wBAAsB,aAAa,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAsDpF;AAMD;;GAEG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAmDtE;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CAe5D;AAMD;;;GAGG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAgQtE;AAMD;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAyKrE;AAMD;;GAEG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAyDlE;AAMD;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,eAAe,GAAG,UAAU,CAsD5D"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/core/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAmBH,OAAO,EAA+B,KAAK,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAInF,OAAO,EAAyC,KAAK,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AACzG,OAAO,EAA0B,KAAK,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,EAAiC,KAAK,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC1F,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEpC,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAEtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEzD,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,EAAE,KAAK,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAGlB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,GAAG,EAAE,OAAO,CAAC;IACb,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,CAAC,EAAE,GAAG,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,iBAAiB,CAAC;IAGnC,WAAW,CAAC,EAAE,GAAG,CAAC;IAGlB,WAAW,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;IAG3D,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,GAAG,CAAC;IACd,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,SAAS,EAAE,MAAM,CAAC;IAGlB,SAAS,CAAC,EAAE;QACV,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IAGF,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,UAAU,CAAC,EAAE,mBAAmB,CAAC;IACjC,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sFAAsF;IACtF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,6DAA6D;IAC7D,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,uDAAuD;AACvD,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,eAAe,CAqDhF;AAMD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CA+F3D;AAMD;;;GAGG;AACH,wBAAsB,aAAa,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAsDpF;AAMD;;GAEG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAsKtE;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CAkB5D;AAMD;;;GAGG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAkRtE;AAMD;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAmPrE;AAMD;;GAEG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAyDlE;AAMD;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,eAAe,GAAG,UAAU,CAqF5D"}