whale-code 6.4.0

package/dist/server/handlers/browser.js

@@ -0,0 +1,517 @@
+// server/handlers/browser.ts — Browser automation via Playwright
+// Supports: navigate, screenshot, extract, click, fill, evaluate, pdf
+import { chromium } from "playwright-core";
+// ============================================================================
+// BROWSER INSTANCE LIFECYCLE
+// ============================================================================
+let browserInstance = null;
+let idleTimer = null;
+const IDLE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
+const PAGE_TIMEOUT_MS = 60_000; // 60 seconds per page operation (increased for CAPTCHA solving)
+const MAX_RESULT_BYTES = 500 * 1024; // 500KB safety cap — context_management handles limits
+const MAX_TEXT_CHARS = 200_000; // 200K chars for text content
+function resetIdleTimer() {
+    if (idleTimer)
+        clearTimeout(idleTimer);
+    idleTimer = setTimeout(async () => {
+        if (browserInstance) {
+            try {
+                await browserInstance.close();
+            }
+            catch { /* browser may already be closed */ }
+            browserInstance = null;
+            console.log("[browser] Closed after idle timeout");
+        }
+    }, IDLE_TIMEOUT_MS);
+}
+async function getBrowser() {
+    if (browserInstance && browserInstance.isConnected()) {
+        resetIdleTimer();
+        return browserInstance;
+    }
+    browserInstance = await chromium.launch({
+        headless: true,
+        executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || undefined,
+        args: [
+            "--no-sandbox",
+            "--disable-setuid-sandbox",
+            "--disable-dev-shm-usage",
+            "--disable-gpu",
+        ],
+    });
+    browserInstance.on("disconnected", () => {
+        browserInstance = null;
+        if (idleTimer) {
+            clearTimeout(idleTimer);
+            idleTimer = null;
+        }
+    });
+    resetIdleTimer();
+    console.log("[browser] Launched new instance");
+    return browserInstance;
+}
+async function withPage(url, waitFor, fn) {
+    const browser = await getBrowser();
+    const context = await browser.newContext({
+        userAgent: "WhaleBot/1.0 (https://whale.app)",
+        viewport: { width: 1280, height: 720 },
+        ignoreHTTPSErrors: true,
+    });
+    context.setDefaultTimeout(PAGE_TIMEOUT_MS);
+    const page = await context.newPage();
+    try {
+        await page.goto(url, { waitUntil: "domcontentloaded", timeout: PAGE_TIMEOUT_MS });
+        // Auto-detect and solve CAPTCHAs after page load
+        const captchaResult = await detectAndSolveCaptcha(page);
+        if (captchaResult.solved) {
+            console.log(`[browser] Solved ${captchaResult.type} CAPTCHA on ${url}`);
+            // Wait for page to update after CAPTCHA solution
+            await page.waitForTimeout(2000);
+        }
+        if (waitFor) {
+            await page.waitForSelector(waitFor, { timeout: PAGE_TIMEOUT_MS });
+        }
+        return await fn(page);
+    }
+    finally {
+        await page.close().catch(() => { });
+        await context.close().catch(() => { });
+    }
+}
+// ============================================================================
+// SSRF PROTECTION
+// ============================================================================
+function isBlockedUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return "Invalid URL";
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+        return `Blocked protocol: ${parsed.protocol}`;
+    }
+    const host = parsed.hostname.toLowerCase();
+    // Block localhost variants
+    if (host === "localhost" || host === "127.0.0.1" || host === "0.0.0.0" ||
+        host === "::1" || host === "[::1]") {
+        return "Blocked: localhost";
+    }
+    // Block private/internal domains
+    if (host.endsWith(".internal") || host.endsWith(".local") ||
+        host === "metadata.google.internal") {
+        return "Blocked: internal/local domain";
+    }
+    // Block private IP ranges
+    if (/^10\./.test(host) ||
+        /^172\.(1[6-9]|2\d|3[01])\./.test(host) ||
+        /^192\.168\./.test(host) ||
+        /^169\.254\./.test(host)) {
+        return "Blocked: private IP range";
+    }
+    // Block IPv6 private
+    if (host.startsWith("fd") || host.startsWith("fc00:") ||
+        host.startsWith("fe80:") || host.includes("::ffff:")) {
+        return "Blocked: IPv6 private range";
+    }
+    return null;
+}
+// ============================================================================
+// TEXT CLEANUP
+// ============================================================================
+function stripScriptsAndStyles(html) {
+    return html
+        .replace(/<script[\s\S]*?<\/script>/gi, "")
+        .replace(/<style[\s\S]*?<\/style>/gi, "")
+        .replace(/<[^>]+>/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+function truncate(text, maxChars) {
+    if (text.length <= maxChars)
+        return text;
+    return text.substring(0, maxChars) + "\n...[truncated]";
+}
+function truncateResult(data) {
+    const json = JSON.stringify(data);
+    if (json.length <= MAX_RESULT_BYTES)
+        return data;
+    // If too large, stringify and truncate
+    return JSON.parse(truncate(json, MAX_RESULT_BYTES));
+}
+// ============================================================================
+// ACTION HANDLERS
+// ============================================================================
+async function actionNavigate(url, waitFor) {
+    return withPage(url, waitFor, async (page) => {
+        const title = await page.title();
+        const html = await page.content();
+        const textContent = stripScriptsAndStyles(html);
+        return {
+            success: true,
+            data: {
+                url: page.url(),
+                title,
+                text: truncate(textContent, MAX_TEXT_CHARS),
+                text_length: textContent.length,
+            },
+        };
+    });
+}
+async function actionScreenshot(url, waitFor) {
+    return withPage(url, waitFor, async (page) => {
+        const buffer = await page.screenshot({ type: "png", fullPage: false });
+        const base64 = buffer.toString("base64");
+        const title = await page.title();
+        return {
+            success: true,
+            data: {
+                url: page.url(),
+                title,
+                screenshot_base64: base64,
+                format: "png",
+                size_bytes: buffer.length,
+            },
+        };
+    });
+}
+async function actionExtract(url, selector, waitFor) {
+    if (!selector) {
+        return { success: false, error: "selector is required for extract action" };
+    }
+    return withPage(url, waitFor, async (page) => {
+        const elements = await page.$$eval(selector, (els) => els.map((el) => ({
+            tag: el.tagName.toLowerCase(),
+            text: el.textContent?.trim() || "",
+            html: el.innerHTML.substring(0, 2000),
+            attributes: Object.fromEntries(Array.from(el.attributes).map((a) => [a.name, a.value])),
+        })));
+        return {
+            success: true,
+            data: {
+                url: page.url(),
+                selector,
+                count: elements.length,
+                elements: elements.slice(0, 50), // Cap at 50 elements
+            },
+        };
+    });
+}
+async function actionClick(url, selector, waitFor) {
+    if (!selector) {
+        return { success: false, error: "selector is required for click action" };
+    }
+    return withPage(url, waitFor, async (page) => {
+        await page.click(selector, { timeout: PAGE_TIMEOUT_MS });
+        // Wait a moment for any navigation or dynamic content
+        await page.waitForTimeout(1000);
+        const title = await page.title();
+        const html = await page.content();
+        const textContent = stripScriptsAndStyles(html);
+        return {
+            success: true,
+            data: {
+                url: page.url(),
+                title,
+                clicked: selector,
+                text: truncate(textContent, MAX_TEXT_CHARS),
+            },
+        };
+    });
+}
+async function actionFill(url, selector, value, waitFor) {
+    if (!selector) {
+        return { success: false, error: "selector is required for fill action" };
+    }
+    if (value === undefined || value === null) {
+        return { success: false, error: "value is required for fill action" };
+    }
+    return withPage(url, waitFor, async (page) => {
+        await page.fill(selector, value, { timeout: PAGE_TIMEOUT_MS });
+        return {
+            success: true,
+            data: {
+                url: page.url(),
+                filled: { selector, value },
+            },
+        };
+    });
+}
+async function actionEvaluate(url, script, waitFor) {
+    if (!script) {
+        return { success: false, error: "script is required for evaluate action" };
+    }
+    return withPage(url, waitFor, async (page) => {
+        const result = await page.evaluate(script);
+        return {
+            success: true,
+            data: {
+                url: page.url(),
+                result: truncateResult(result),
+            },
+        };
+    });
+}
+async function actionPdf(url, waitFor) {
+    return withPage(url, waitFor, async (page) => {
+        const buffer = await page.pdf({
+            format: "A4",
+            printBackground: true,
+            margin: { top: "1cm", bottom: "1cm", left: "1cm", right: "1cm" },
+        });
+        const base64 = buffer.toString("base64");
+        const title = await page.title();
+        return {
+            success: true,
+            data: {
+                url: page.url(),
+                title,
+                pdf_base64: base64,
+                format: "A4",
+                size_bytes: buffer.length,
+            },
+        };
+    });
+}
+// ============================================================================
+// PDF FROM HTML (reusable by documents handler)
+// ============================================================================
+export async function generatePdfFromHtml(html, options) {
+    // P0 FIX: Strip dangerous tags from user HTML before rendering to prevent
+    // script execution, SSRF via iframes, and local file reads
+    const sanitizedHtml = html
+        .replace(/<script[\s\S]*?<\/script>/gi, "")
+        .replace(/<script[^>]*\/?>/gi, "")
+        .replace(/<iframe[\s\S]*?<\/iframe>/gi, "")
+        .replace(/<iframe[^>]*\/?>/gi, "")
+        .replace(/<object[\s\S]*?<\/object>/gi, "")
+        .replace(/<object[^>]*\/?>/gi, "")
+        .replace(/<embed[^>]*\/?>/gi, "")
+        .replace(/<link[^>]*\/?>/gi, "");
+    const browser = await getBrowser();
+    const context = await browser.newContext({ viewport: { width: 1280, height: 720 } });
+    const page = await context.newPage();
+    try {
+        // P0 FIX: Block all network requests to prevent SSRF from headless Chromium.
+        // Only allow data: URIs (inline images/fonts) and about:blank.
+        await page.route('**/*', (route) => {
+            const url = route.request().url();
+            if (url.startsWith('data:') || url === 'about:blank') {
+                route.continue();
+            }
+            else {
+                route.abort();
+            }
+        });
+        // P0 FIX: Use domcontentloaded instead of networkidle to avoid waiting for
+        // (now-blocked) network requests and reduce attack surface
+        await page.setContent(sanitizedHtml, { waitUntil: "domcontentloaded", timeout: PAGE_TIMEOUT_MS });
+        const buffer = await page.pdf({
+            format: options?.format || "A4",
+            printBackground: true,
+            landscape: options?.landscape || false,
+            margin: { top: "1cm", bottom: "1cm", left: "1cm", right: "1cm" },
+        });
+        return Buffer.from(buffer);
+    }
+    finally {
+        await page.close().catch(() => { });
+        await context.close().catch(() => { });
+    }
+}
+async function callCapMonster(taskType, params) {
+    const apiKey = process.env.CAPMONSTER_API_KEY;
+    if (!apiKey)
+        throw new Error("CAPMONSTER_API_KEY not configured");
+    const createResp = await fetch("https://api.capmonster.cloud/createTask", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ clientKey: apiKey, task: { type: taskType, ...params } }),
+    });
+    const createData = await createResp.json();
+    if (createData.errorId)
+        throw new Error(`CapMonster create: ${createData.errorDescription}`);
+    const taskId = createData.taskId;
+    // Poll for result (max 120s, 2s intervals)
+    for (let i = 0; i < 60; i++) {
+        await new Promise(r => setTimeout(r, 2000));
+        const resultResp = await fetch("https://api.capmonster.cloud/getTaskResult", {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ clientKey: apiKey, taskId }),
+        });
+        const resultData = await resultResp.json();
+        if (resultData.status === "ready") {
+            return resultData.solution?.gRecaptchaResponse || resultData.solution?.token || resultData.solution?.cf_clearance || "";
+        }
+        if (resultData.errorId)
+            throw new Error(`CapMonster result: ${resultData.errorDescription}`);
+    }
+    throw new Error("CapMonster: timeout waiting for solution (120s)");
+}
+async function detectAndSolveCaptcha(page) {
+    try {
+        // Skip if no API key configured
+        if (!process.env.CAPMONSTER_API_KEY)
+            return { solved: false };
+        const pageUrl = page.url();
+        // Detect reCAPTCHA v2
+        const recaptchaKey = await page.evaluate(() => {
+            const el = document.querySelector('.g-recaptcha[data-sitekey], [data-sitekey]');
+            if (el && el.classList.contains('g-recaptcha'))
+                return el.getAttribute('data-sitekey');
+            // Also check for reCAPTCHA iframe
+            const iframe = document.querySelector('iframe[src*="recaptcha"]');
+            if (iframe) {
+                const src = iframe.getAttribute('src') || '';
+                const match = src.match(/[?&]k=([^&]+)/);
+                return match ? match[1] : null;
+            }
+            return null;
+        }).catch(() => null);
+        if (recaptchaKey) {
+            console.log(`[browser] Detected reCAPTCHA v2 on ${pageUrl}`);
+            const token = await callCapMonster("RecaptchaV2TaskProxyless", {
+                websiteURL: pageUrl,
+                websiteKey: recaptchaKey,
+            });
+            await page.evaluate((t) => {
+                const textarea = document.getElementById('g-recaptcha-response');
+                if (textarea) {
+                    textarea.style.display = 'block';
+                    textarea.value = t;
+                }
+                // Try triggering the callback
+                const win = window;
+                try {
+                    const cfg = win.___grecaptcha_cfg;
+                    if (cfg?.clients) {
+                        const clients = cfg.clients;
+                        const firstKey = Object.keys(clients)[0];
+                        const first = firstKey ? clients[firstKey] : null;
+                        if (first) {
+                            const findCallback = (obj, depth = 0) => {
+                                if (depth > 4 || !obj || typeof obj !== 'object')
+                                    return null;
+                                for (const val of Object.values(obj)) {
+                                    if (typeof val === 'function')
+                                        return val;
+                                    const found = findCallback(val, depth + 1);
+                                    if (found)
+                                        return found;
+                                }
+                                return null;
+                            };
+                            const cb = findCallback(first);
+                            if (cb)
+                                cb(t);
+                        }
+                    }
+                }
+                catch { /* callback search failed, form submit will work */ }
+            }, token);
+            return { solved: true, type: "reCAPTCHA v2" };
+        }
+        // Detect hCaptcha
+        const hcaptchaKey = await page.evaluate(() => {
+            const el = document.querySelector('.h-captcha[data-sitekey]');
+            return el?.getAttribute('data-sitekey') || null;
+        }).catch(() => null);
+        if (hcaptchaKey) {
+            console.log(`[browser] Detected hCaptcha on ${pageUrl}`);
+            const token = await callCapMonster("HCaptchaTaskProxyless", {
+                websiteURL: pageUrl,
+                websiteKey: hcaptchaKey,
+            });
+            await page.evaluate((t) => {
+                const textarea = document.querySelector('[name="h-captcha-response"], [name="g-recaptcha-response"]');
+                if (textarea) {
+                    textarea.value = t;
+                }
+                const win = window;
+                if (typeof win.hcaptchaCallback === 'function') {
+                    win.hcaptchaCallback(t);
+                }
+            }, token);
+            return { solved: true, type: "hCaptcha" };
+        }
+        // Detect Cloudflare Turnstile
+        const turnstileKey = await page.evaluate(() => {
+            const el = document.querySelector('.cf-turnstile[data-sitekey]');
+            return el?.getAttribute('data-sitekey') || null;
+        }).catch(() => null);
+        if (turnstileKey) {
+            console.log(`[browser] Detected Turnstile on ${pageUrl}`);
+            const token = await callCapMonster("TurnstileTaskProxyless", {
+                websiteURL: pageUrl,
+                websiteKey: turnstileKey,
+            });
+            await page.evaluate((t) => {
+                const input = document.querySelector('[name="cf-turnstile-response"]');
+                if (input)
+                    input.value = t;
+                const win = window;
+                if (typeof win.turnstileCallback === 'function') {
+                    win.turnstileCallback(t);
+                }
+            }, token);
+            return { solved: true, type: "Turnstile" };
+        }
+        return { solved: false };
+    }
+    catch (err) {
+        console.log(`[browser] CAPTCHA detection/solving failed: ${err instanceof Error ? err.message : String(err)}`);
+        return { solved: false };
+    }
+}
+// ============================================================================
+// MAIN HANDLER
+// ============================================================================
+export async function handleBrowser(_sb, args, _storeId) {
+    const action = args.action;
+    const url = args.url;
+    const selector = args.selector;
+    const value = args.value;
+    const script = args.script;
+    const waitFor = args.wait_for;
+    if (!action) {
+        return { success: false, error: "action is required" };
+    }
+    if (!url) {
+        return { success: false, error: "url is required" };
+    }
+    // SSRF check
+    const blocked = isBlockedUrl(url);
+    if (blocked) {
+        return { success: false, error: blocked };
+    }
+    try {
+        switch (action) {
+            case "navigate":
+                return await actionNavigate(url, waitFor);
+            case "screenshot":
+                return await actionScreenshot(url, waitFor);
+            case "extract":
+                return await actionExtract(url, selector, waitFor);
+            case "click":
+                return await actionClick(url, selector, waitFor);
+            case "fill":
+                return await actionFill(url, selector, value, waitFor);
+            case "evaluate":
+                return await actionEvaluate(url, script, waitFor);
+            case "pdf":
+                return await actionPdf(url, waitFor);
+            default:
+                return { success: false, error: `Unknown browser action: ${action}. Valid: navigate, screenshot, extract, click, fill, evaluate, pdf` };
+        }
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        // Clean up common Playwright error noise
+        const cleanMessage = message
+            .replace(/=========================== logs ===========================[^]*?============================================================/gs, "")
+            .trim();
+        return { success: false, error: `Browser error: ${cleanMessage.substring(0, 500)}` };
+    }
+}

package/dist/server/handlers/catalog.d.ts

@@ -0,0 +1,99 @@
+import type { SupabaseClient } from "@supabase/supabase-js";
+export declare function handleProducts(sb: SupabaseClient, args: Record<string, unknown>, storeId?: string): Promise<{
+    success: boolean;
+    error: string;
+    data?: undefined;
+    count?: undefined;
+} | {
+    data: any[];
+    limit?: {} | undefined;
+    success: boolean;
+    total: number | null;
+    count: number;
+    offset: number;
+    error?: undefined;
+} | {
+    success: boolean;
+    data: any;
+    error?: undefined;
+    count?: undefined;
+} | {
+    success: boolean;
+    count: number;
+    data: {
+        id: any;
+        name: any;
+        slug: any;
+        description: any;
+        icon: any;
+        parent_id: any;
+        display_order: any;
+        is_active: any;
+        featured: any;
+        product_count: any;
+        catalog_id: any;
+        field_schema_id: any;
+        created_at: any;
+    }[];
+    error?: undefined;
+} | {
+    success: boolean;
+    count: number;
+    data: {
+        id: any;
+        name: any;
+        slug: any;
+        description: any;
+        icon: any;
+        fields: any;
+        is_public: any;
+        is_active: any;
+        catalog_id: any;
+        store_id: any;
+        install_count: any;
+        created_at: any;
+    }[];
+    error?: undefined;
+} | {
+    success: boolean;
+    count: number;
+    data: {
+        id: any;
+        name: any;
+        slug: any;
+        description: any;
+        tiers: any;
+        quality_tier: any;
+        is_public: any;
+        is_active: any;
+        catalog_id: any;
+        store_id: any;
+        install_count: any;
+        created_at: any;
+    }[];
+    error?: undefined;
+} | {
+    success: boolean;
+    count: number;
+    data: {
+        id: any;
+        name: any;
+        slug: any;
+        description: any;
+        vertical: any;
+        is_active: any;
+        is_default: any;
+        display_order: any;
+        created_at: any;
+    }[];
+    error?: undefined;
+}>;
+export declare function handleCollections(sb: SupabaseClient, args: Record<string, unknown>, storeId?: string): Promise<{
+    success: boolean;
+    error: string;
+    data?: undefined;
+} | {
+    success: boolean;
+    data: any;
+    error?: undefined;
+}>;