toolpack-sdk 0.1.0-SNAPSHOT

package/dist/tools/web-tools/tools/scrape/index.js

@@ -0,0 +1,249 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.webScrapeTool = void 0;
+const cheerio = __importStar(require("cheerio"));
+const schema_js_1 = require("./schema.js");
+const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36';
+const JUNK_SELECTORS = ['script', 'style', 'nav', 'footer', 'header', 'iframe', '.ads', '.sidebar', 'noscript'];
+const CONTENT_SELECTORS = ['article', 'main', '.content', '#content', '[role="main"]', 'body'];
+const MIN_CONTENT_LENGTH = 200;
+async function execute(args) {
+    const url = args.url;
+    const selector = args.selector;
+    const section = args.section;
+    const format = (args.format || 'text');
+    const maxLength = (args.max_length || 6000);
+    const timeout = (args.timeout || 30000);
+    if (!url) {
+        throw new Error('url is required');
+    }
+    if (!url.startsWith('http://') && !url.startsWith('https://')) {
+        throw new Error('url must start with http:// or https://');
+    }
+    const controller = new AbortController();
+    const abortTimeout = setTimeout(() => controller.abort(), timeout);
+    let response;
+    try {
+        response = await fetch(url, {
+            method: 'GET',
+            headers: { 'User-Agent': USER_AGENT },
+            signal: controller.signal,
+        });
+    }
+    catch (e) {
+        if (e.name === 'AbortError')
+            throw new Error(`Request timed out after ${timeout}ms`);
+        throw e;
+    }
+    finally {
+        clearTimeout(abortTimeout);
+    }
+    if (!response.ok) {
+        throw new Error(`Failed to fetch ${url}: HTTP ${response.status} ${response.statusText}`);
+    }
+    const html = await response.text();
+    const $ = cheerio.load(html);
+    // Remove junk elements
+    for (const junk of JUNK_SELECTORS) {
+        $(junk).remove();
+    }
+    if (format === 'tables') {
+        const tables = [];
+        $('table').each((i, tableEl) => {
+            const tableData = [];
+            const headers = [];
+            // Extract headers if present
+            $(tableEl).find('th').each((_, th) => {
+                headers.push($(th).text().trim());
+            });
+            // Extract rows
+            $(tableEl).find('tr').each((_, tr) => {
+                const rowData = headers.length > 0 ? {} : [];
+                const cells = $(tr).find('td');
+                if (cells.length > 0) {
+                    cells.each((colIdx, td) => {
+                        const text = $(td).text().trim();
+                        if (headers.length > 0 && headers[colIdx]) {
+                            rowData[headers[colIdx]] = text;
+                        }
+                        else {
+                            rowData.push(text);
+                        }
+                    });
+                    tableData.push(rowData);
+                }
+            });
+            if (tableData.length > 0) {
+                tables.push({ id: `Table ${i + 1}`, headers: headers.length > 0 ? headers : undefined, rows: tableData });
+            }
+        });
+        if (tables.length === 0) {
+            return `No tables found on ${url}`;
+        }
+        return JSON.stringify(tables, null, 2);
+    }
+    let content = '';
+    // Section-based extraction (natural language targeting)
+    if (section) {
+        const sectionLower = section.toLowerCase();
+        let foundHeadingEl = null;
+        let headingLevel = 0;
+        // Find heading containing the section text
+        $('h1, h2, h3, h4, h5, h6').each((_, element) => {
+            const headingText = $(element).text().toLowerCase();
+            if (headingText.includes(sectionLower)) {
+                foundHeadingEl = element;
+                headingLevel = parseInt(element.tagName.charAt(1));
+                return false; // break
+            }
+            return undefined;
+        });
+        if (foundHeadingEl) {
+            const foundHeading = $(foundHeadingEl);
+            // Extract content from this heading until the next same-level heading
+            const sectionContent = [];
+            let current = foundHeading.next();
+            while (current.length > 0) {
+                const tagName = current.prop('tagName')?.toLowerCase();
+                // Stop if we hit another heading of same or higher level
+                if (tagName && /^h[1-6]$/.test(tagName)) {
+                    const currentLevel = parseInt(tagName.charAt(1));
+                    if (currentLevel <= headingLevel) {
+                        break;
+                    }
+                }
+                // Get text from this element (includes nested content)
+                const text = current.text().trim();
+                if (text) {
+                    sectionContent.push(text);
+                }
+                current = current.next();
+            }
+            if (sectionContent.length > 0) {
+                content = sectionContent.join('\n\n').replace(/\s+/g, ' ').trim();
+                content = `[Section: "${section}"]\n\n${content}`;
+            }
+            else {
+                content = `[Note: Found heading "${section}" but no content below it. Falling back to full page.]\n\n`;
+            }
+        }
+        else {
+            content = `[Note: Section "${section}" not found. Falling back to full page.]
+
+`;
+        }
+    }
+    // Only run selector/auto-detect if section extraction didn't find content
+    const needsFallback = !content || content.includes('Falling back to full page');
+    if (needsFallback) {
+        // Clear the fallback message if present
+        if (content && content.includes('Falling back to full page')) {
+            content = '';
+        }
+        if (selector) {
+            // User-specified selector
+            const element = $(selector);
+            if (element.length > 0) {
+                content = element.text().replace(/\s+/g, ' ').trim();
+            }
+            else {
+                // Selector not found — fall back to auto-detect
+                for (const sel of CONTENT_SELECTORS) {
+                    const element = $(sel);
+                    if (element.length > 0) {
+                        const text = element.text().replace(/\s+/g, ' ').trim();
+                        if (text.length > MIN_CONTENT_LENGTH) {
+                            content = text;
+                            break;
+                        }
+                    }
+                }
+                if (content) {
+                    content = `[Note: Selector "${selector}" not found. Showing auto-detected main content instead.]\n\n${content}`;
+                }
+                else {
+                    return `No element found matching selector "${selector}" and could not auto-detect main content from ${url}`;
+                }
+            }
+        }
+        else {
+            // Auto-detect main content using cascade
+            for (const sel of CONTENT_SELECTORS) {
+                const element = $(sel);
+                if (element.length > 0) {
+                    const text = element.text().replace(/\s+/g, ' ').trim();
+                    if (text.length > MIN_CONTENT_LENGTH) {
+                        content = text;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    // Final fallback: grab whatever text remains (even if minimal)
+    if (!content) {
+        content = $('body').text().replace(/\s+/g, ' ').trim();
+        if (content) {
+            content = `[Note: Could not detect main content area. Showing full page text (may include navigation).]\n\n${content}`;
+        }
+    }
+    if (!content) {
+        return `Could not extract any content from ${url}`;
+    }
+    // Intelligent chunking: if content is too large, guide the AI to use better strategies
+    if (content.length > maxLength) {
+        const truncated = content.substring(0, maxLength);
+        return `[Warning: Content exceeds ${maxLength} chars (actual: ${content.length} chars). Showing first ${maxLength} chars only.]
+
+RECOMMENDATION: Use web.map to see page structure, then use web.scrape with section parameter to extract specific sections.
+
+Page content from ${url}:
+
+${truncated}
+
+... [Content truncated. ${content.length - maxLength} chars remaining. Use section parameter to extract specific sections.]`;
+    }
+    return `Page content from ${url}:\n\n${content}`;
+}
+exports.webScrapeTool = {
+    name: schema_js_1.name,
+    displayName: schema_js_1.displayName,
+    description: schema_js_1.description,
+    parameters: schema_js_1.parameters,
+    category: schema_js_1.category,
+    execute,
+};
+//# sourceMappingURL=index.js.map

package/dist/tools/web-tools/tools/scrape/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/tools/web-tools/tools/scrape/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAmC;AAEnC,2CAAmF;AAEnF,MAAM,UAAU,GAAG,qHAAqH,CAAC;AAEzI,MAAM,cAAc,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;AAChH,MAAM,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;AAC/F,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,KAAK,UAAU,OAAO,CAAC,IAAyB;IAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAa,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAA8B,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,CAAC,OAA6B,CAAC;IACnD,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAsB,CAAC;IAC5D,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAW,CAAC;IACtD,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,KAAK,CAAW,CAAC;IAElD,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IAEnE,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACD,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACxB,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE;YACrC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC5B,CAAC,CAAC;IACP,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,OAAO,IAAI,CAAC,CAAC;QACrF,MAAM,CAAC,CAAC;IACZ,CAAC;YAAS,CAAC;QACP,YAAY,CAAC,YAAY,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,UAAU,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,uBAAuB;IACvB,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAChC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;IACrB,CAAC;IAED,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACtB,MAAM,MAAM,GAAU,EAAE,CAAC;QACzB,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YAC3B,MAAM,SAAS,GAAU,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAa,EAAE,CAAC;YAE7B,6BAA6B;YAC7B,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YAEH,eAAe;YACf,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBACjC,MAAM,OAAO,GAAsC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChF,MAAM,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAE/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnB,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE;wBACtB,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;wBACjC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;4BACvC,OAAkC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC;wBAChE,CAAC;6BAAM,CAAC;4BACH,OAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBACrC,CAAC;oBACL,CAAC,CAAC,CAAC;oBACH,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC5B,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YAC9G,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,sBAAsB,GAAG,EAAE,CAAC;QACvC,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,wDAAwD;IACxD,IAAI,OAAO,EAAE,CAAC;QACV,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAC3C,IAAI,cAAc,GAAQ,IAAI,CAAC;QAC/B,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,2CAA2C;QAC3C,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YAC5C,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACpD,IAAI,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACrC,cAAc,GAAG,OAAO,CAAC;gBACzB,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnD,OAAO,KAAK,CAAC,CAAC,QAAQ;YAC1B,CAAC;YACD,OAAO,SAAS,CAAC;QACrB,CAAC,CAAC,CAAC;QAEH,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,YAAY,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC;YACvC,sEAAsE;YACtE,MAAM,cAAc,GAAa,EAAE,CAAC;YACpC,IAAI,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC;YAElC,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;gBAEvD,yDAAyD;gBACzD,IAAI,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBACtC,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;oBACjD,IAAI,YAAY,IAAI,YAAY,EAAE,CAAC;wBAC/B,MAAM;oBACV,CAAC;gBACL,CAAC;gBAED,uDAAuD;gBACvD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,IAAI,EAAE,CAAC;oBACP,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC9B,CAAC;gBACD,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAC7B,CAAC;YAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBAClE,OAAO,GAAG,cAAc,OAAO,SAAS,OAAO,EAAE,CAAC;YACtD,CAAC;iBAAM,CAAC;gBACJ,OAAO,GAAG,yBAAyB,OAAO,4DAA4D,CAAC;YAC3G,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,OAAO,GAAG,mBAAmB,OAAO;;CAE/C,CAAC;QACM,CAAC;IACL,CAAC;IAED,0EAA0E;IAC1E,MAAM,aAAa,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,CAAC;IAChF,IAAI,aAAa,EAAE,CAAC;QAChB,wCAAwC;QACxC,IAAI,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;YAC3D,OAAO,GAAG,EAAE,CAAC;QACjB,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACX,0BAA0B;YAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC5B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YACzD,CAAC;iBAAM,CAAC;gBACJ,gDAAgD;gBAChD,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;oBAClC,MAAM,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;oBACvB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACrB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;wBACxD,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;4BACnC,OAAO,GAAG,IAAI,CAAC;4BACf,MAAM;wBACV,CAAC;oBACL,CAAC;gBACL,CAAC;gBACD,IAAI,OAAO,EAAE,CAAC;oBACV,OAAO,GAAG,oBAAoB,QAAQ,gEAAgE,OAAO,EAAE,CAAC;gBACpH,CAAC;qBAAM,CAAC;oBACJ,OAAO,uCAAuC,QAAQ,iDAAiD,GAAG,EAAE,CAAC;gBACjH,CAAC;YACL,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,yCAAyC;YACzC,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;gBACvB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;oBACxD,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;wBACnC,OAAO,GAAG,IAAI,CAAC;wBACf,MAAM;oBACV,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,+DAA+D;IAC/D,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,EAAE,CAAC;YACV,OAAO,GAAG,mGAAmG,OAAO,EAAE,CAAC;QAC3H,CAAC;IACL,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,OAAO,sCAAsC,GAAG,EAAE,CAAC;IACvD,CAAC;IAED,uFAAuF;IACvF,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAClD,OAAO,6BAA6B,SAAS,mBAAmB,OAAO,CAAC,MAAM,0BAA0B,SAAS;;;;oBAIrG,GAAG;;EAErB,SAAS;;0BAEe,OAAO,CAAC,MAAM,GAAG,SAAS,wEAAwE,CAAC;IACzH,CAAC;IAED,OAAO,qBAAqB,GAAG,QAAQ,OAAO,EAAE,CAAC;AACrD,CAAC;AAEY,QAAA,aAAa,GAAmB;IACzC,IAAI,EAAJ,gBAAI;IACJ,WAAW,EAAX,uBAAW;IACX,WAAW,EAAX,uBAAW;IACX,UAAU,EAAV,sBAAU;IACV,QAAQ,EAAR,oBAAQ;IACR,OAAO;CACV,CAAC"}

package/dist/tools/web-tools/tools/scrape/schema.d.ts

@@ -0,0 +1,7 @@
+import { ToolParameters } from '../../../types.js';
+export declare const name = "web.scrape";
+export declare const displayName = "Scrape";
+export declare const description = "Extract clean text content from a webpage. RECOMMENDED WORKFLOW: Use web.map first to see page structure, then use section parameter to extract specific sections. Strips scripts, styles, navigation, and other junk. By default, auto-detects and extracts the main article/content area.";
+export declare const category = "network";
+export declare const parameters: ToolParameters;
+//# sourceMappingURL=schema.d.ts.map

package/dist/tools/web-tools/tools/scrape/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../../../src/tools/web-tools/tools/scrape/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,eAAO,MAAM,IAAI,eAAe,CAAC;AACjC,eAAO,MAAM,WAAW,WAAW,CAAC;AACpC,eAAO,MAAM,WAAW,gSAAgS,CAAC;AACzT,eAAO,MAAM,QAAQ,YAAY,CAAC;AAElC,eAAO,MAAM,UAAU,EAAE,cAiCxB,CAAC"}

package/dist/tools/web-tools/tools/scrape/schema.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parameters = exports.category = exports.description = exports.displayName = exports.name = void 0;
+exports.name = 'web.scrape';
+exports.displayName = 'Scrape';
+exports.description = 'Extract clean text content from a webpage. RECOMMENDED WORKFLOW: Use web.map first to see page structure, then use section parameter to extract specific sections. Strips scripts, styles, navigation, and other junk. By default, auto-detects and extracts the main article/content area.';
+exports.category = 'network';
+exports.parameters = {
+    type: 'object',
+    properties: {
+        url: {
+            type: 'string',
+            description: 'The URL to scrape',
+        },
+        section: {
+            type: 'string',
+            description: 'Optional section name to extract (e.g., "talks", "about", "experience"). Finds the heading containing this text and extracts content until the next same-level heading. Use web.map first to discover available sections.',
+        },
+        format: {
+            type: 'string',
+            description: 'Output format: "text" (clean readable text) or "tables" (extract table data as JSON array). Default: "text"',
+            enum: ['text', 'tables'],
+            default: 'text',
+        },
+        selector: {
+            type: 'string',
+            description: 'Optional CSS selector to target a specific element. Only use if you know the exact selector exists.',
+        },
+        max_length: {
+            type: 'integer',
+            description: 'Maximum characters to return (default: 6000). Keep small (3000-6000) to avoid context limits.',
+            default: 6000,
+        },
+        timeout: {
+            type: 'integer',
+            description: 'Timeout in milliseconds (default: 30000)',
+            default: 30000,
+        },
+    },
+    required: ['url'],
+};
+//# sourceMappingURL=schema.js.map

package/dist/tools/web-tools/tools/scrape/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../../../src/tools/web-tools/tools/scrape/schema.ts"],"names":[],"mappings":";;;AAEa,QAAA,IAAI,GAAG,YAAY,CAAC;AACpB,QAAA,WAAW,GAAG,QAAQ,CAAC;AACvB,QAAA,WAAW,GAAG,6RAA6R,CAAC;AAC5S,QAAA,QAAQ,GAAG,SAAS,CAAC;AAErB,QAAA,UAAU,GAAmB;IACtC,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE;QACR,GAAG,EAAE;YACD,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,mBAAmB;SACnC;QACD,OAAO,EAAE;YACL,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2NAA2N;SAC3O;QACD,MAAM,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,6GAA6G;YAC1H,IAAI,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;YACxB,OAAO,EAAE,MAAM;SAClB;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,qGAAqG;SACrH;QACD,UAAU,EAAE;YACR,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,+FAA+F;YAC5G,OAAO,EAAE,IAAI;SAChB;QACD,OAAO,EAAE;YACL,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,0CAA0C;YACvD,OAAO,EAAE,KAAK;SACjB;KACJ;IACD,QAAQ,EAAE,CAAC,KAAK,CAAC;CACpB,CAAC"}

package/dist/tools/web-tools/tools/screenshot/index.d.ts

@@ -0,0 +1,3 @@
+import { ToolDefinition } from '../../../types.js';
+export declare const webScreenshotTool: ToolDefinition;
+//# sourceMappingURL=index.d.ts.map

package/dist/tools/web-tools/tools/screenshot/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/tools/web-tools/tools/screenshot/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAoDnD,eAAO,MAAM,iBAAiB,EAAE,cAO/B,CAAC"}

package/dist/tools/web-tools/tools/screenshot/index.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.webScreenshotTool = void 0;
+const schema_js_1 = require("./schema.js");
+const puppeteer_loader_js_1 = require("../../utils/puppeteer-loader.js");
+const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36';
+async function execute(args) {
+    const url = args.url;
+    const format = (args.format || 'html');
+    const viewport = args.viewport;
+    const timeout = (args.timeout || 30000);
+    if (!url) {
+        throw new Error('url is required');
+    }
+    const puppeteer = await (0, puppeteer_loader_js_1.loadPuppeteer)();
+    const browser = await puppeteer.launch({
+        headless: true,
+        args: ['--no-sandbox', '--disable-setuid-sandbox']
+    });
+    try {
+        const page = await browser.newPage();
+        await page.setUserAgent(USER_AGENT);
+        await page.setViewport({
+            width: viewport?.width || 1280,
+            height: viewport?.height || 800
+        });
+        await page.goto(url, { waitUntil: 'networkidle2', timeout });
+        if (format === 'png') {
+            const buffer = await page.screenshot({ type: 'png', fullPage: true });
+            let base64;
+            // Puppeteer <20 returns Buffer, >=20 Uint8Array. Handle both gracefully
+            if (Buffer.isBuffer(buffer)) {
+                base64 = buffer.toString('base64');
+            }
+            else {
+                base64 = Buffer.from(buffer).toString('base64');
+            }
+            return `data:image/png;base64,${base64}`;
+        }
+        else {
+            const html = await page.content();
+            return html;
+        }
+    }
+    finally {
+        await browser.close();
+    }
+}
+exports.webScreenshotTool = {
+    name: schema_js_1.name,
+    displayName: schema_js_1.displayName,
+    description: schema_js_1.description,
+    parameters: schema_js_1.parameters,
+    category: schema_js_1.category,
+    execute,
+};
+//# sourceMappingURL=index.js.map

package/dist/tools/web-tools/tools/screenshot/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/tools/web-tools/tools/screenshot/index.ts"],"names":[],"mappings":";;;AACA,2CAAmF;AACnF,yEAAgE;AAEhE,MAAM,UAAU,GAAG,qHAAqH,CAAC;AAEzI,KAAK,UAAU,OAAO,CAAC,IAAyB;IAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAa,CAAC;IAC/B,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAmB,CAAC;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAA2D,CAAC;IAClF,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,KAAK,CAAW,CAAC;IAElD,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,IAAA,mCAAa,GAAE,CAAC;IACxC,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC;QACnC,QAAQ,EAAE,IAAI;QACd,IAAI,EAAE,CAAC,cAAc,EAAE,0BAA0B,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QAEpC,MAAM,IAAI,CAAC,WAAW,CAAC;YACnB,KAAK,EAAE,QAAQ,EAAE,KAAK,IAAI,IAAI;YAC9B,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG;SAClC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QAE7D,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,IAAI,MAAM,CAAC;YACX,wEAAwE;YACxE,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACJ,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACpD,CAAC;YACD,OAAO,yBAAyB,MAAM,EAAE,CAAC;QAC7C,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;YAAS,CAAC;QACP,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACL,CAAC;AAEY,QAAA,iBAAiB,GAAmB;IAC7C,IAAI,EAAJ,gBAAI;IACJ,WAAW,EAAX,uBAAW;IACX,WAAW,EAAX,uBAAW;IACX,UAAU,EAAV,sBAAU;IACV,QAAQ,EAAR,oBAAQ;IACR,OAAO;CACV,CAAC"}

package/dist/tools/web-tools/tools/screenshot/schema.d.ts

@@ -0,0 +1,7 @@
+import { ToolParameters } from '../../../types.js';
+export declare const name = "web.screenshot";
+export declare const displayName = "Screenshot";
+export declare const description = "Render a page with headless browser and return screenshot Base64 PNG or rendered HTML.";
+export declare const category = "network";
+export declare const parameters: ToolParameters;
+//# sourceMappingURL=schema.d.ts.map

package/dist/tools/web-tools/tools/screenshot/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../../../src/tools/web-tools/tools/screenshot/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,eAAO,MAAM,IAAI,mBAAmB,CAAC;AACrC,eAAO,MAAM,WAAW,eAAe,CAAC;AACxC,eAAO,MAAM,WAAW,2FAA2F,CAAC;AACpH,eAAO,MAAM,QAAQ,YAAY,CAAC;AAElC,eAAO,MAAM,UAAU,EAAE,cA4BxB,CAAC"}

package/dist/tools/web-tools/tools/screenshot/schema.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parameters = exports.category = exports.description = exports.displayName = exports.name = void 0;
+exports.name = 'web.screenshot';
+exports.displayName = 'Screenshot';
+exports.description = 'Render a page with headless browser and return screenshot Base64 PNG or rendered HTML.';
+exports.category = 'network';
+exports.parameters = {
+    type: 'object',
+    properties: {
+        url: {
+            type: 'string',
+            description: 'The URL to capture',
+        },
+        format: {
+            type: 'string',
+            description: 'Output format: "html" or "png" (default: "html")',
+            enum: ['html', 'png'],
+            default: 'html',
+        },
+        viewport: {
+            type: 'object',
+            description: 'Optional viewport { width, height } (default: { width: 1280, height: 800 })',
+            properties: {
+                width: { type: 'integer' },
+                height: { type: 'integer' }
+            }
+        },
+        timeout: {
+            type: 'integer',
+            description: 'Timeout in milliseconds (default: 30000)',
+            default: 30000,
+        },
+    },
+    required: ['url'],
+};
+//# sourceMappingURL=schema.js.map

package/dist/tools/web-tools/tools/screenshot/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../../../src/tools/web-tools/tools/screenshot/schema.ts"],"names":[],"mappings":";;;AAEa,QAAA,IAAI,GAAG,gBAAgB,CAAC;AACxB,QAAA,WAAW,GAAG,YAAY,CAAC;AAC3B,QAAA,WAAW,GAAG,wFAAwF,CAAC;AACvG,QAAA,QAAQ,GAAG,SAAS,CAAC;AAErB,QAAA,UAAU,GAAmB;IACtC,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE;QACR,GAAG,EAAE;YACD,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,oBAAoB;SACpC;QACD,MAAM,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,kDAAkD;YAC/D,IAAI,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC;YACrB,OAAO,EAAE,MAAM;SAClB;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,6EAA6E;YAC1F,UAAU,EAAE;gBACR,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC1B,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;aAC9B;SACJ;QACD,OAAO,EAAE;YACL,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,0CAA0C;YACvD,OAAO,EAAE,KAAK;SACjB;KACJ;IACD,QAAQ,EAAE,CAAC,KAAK,CAAC;CACpB,CAAC"}

package/dist/tools/web-tools/tools/search/index.d.ts

@@ -0,0 +1,3 @@
+import { ToolDefinition } from '../../../types.js';
+export declare const webSearchTool: ToolDefinition;
+//# sourceMappingURL=index.d.ts.map

package/dist/tools/web-tools/tools/search/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/tools/web-tools/tools/search/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AA6OnD,eAAO,MAAM,aAAa,EAAE,cAO3B,CAAC"}