skillshield 1.0.0

package/dist/tools/web-engine.js

@@ -0,0 +1,802 @@
+/**
+ * SkillKit Native Web Intelligence Engine
+ * =========================================
+ * Zero external API dependencies. No Tavily, no SerpAPI, no paid keys.
+ * SkillKit builds its own web intelligence from scratch.
+ *
+ * 5 capabilities (inspired by Tavily's architecture, built natively):
+ *   1. SEARCH   — Multi-source web search (DuckDuckGo HTML + Google fallback + SearXNG)
+ *   2. EXTRACT  — Content extraction from any URL (Readability-style + metadata)
+ *   3. CRAWL    — Deep website crawling with link following and depth control
+ *   4. RESEARCH — Multi-step deep research with AI-powered synthesis
+ *   5. MAP      — Site URL structure discovery (sitemap.xml + link harvesting)
+ *
+ * How it works WITHOUT paid APIs:
+ *   - Search: Scrapes DuckDuckGo HTML (no API key needed), parses results
+ *   - Extract: Fetches raw HTML, strips boilerplate via content-density algorithm
+ *   - Crawl: BFS link-follower with domain scoping, depth limits, robots.txt respect
+ *   - Research: Orchestrates search → extract → synthesize in multiple rounds
+ *   - Map: Parses sitemap.xml + discovers links via crawl (URL-only mode)
+ *
+ * @module web-engine
+ */
+// ─── Configuration ──────────────────────────────────────────────────────────
+const USER_AGENT = 'SkillKit/1.0 (https://github.com/artefactforge/skillkit)';
+const FETCH_TIMEOUT = 15000;
+const MAX_CONTENT_LENGTH = 500000; // 500KB max per page
+const MAX_REDIRECTS = 5;
+// ─── HTTP Helpers ───────────────────────────────────────────────────────────
+/**
+ * Robust fetch with timeout, redirect following, and User-Agent.
+ */
+async function safeFetch(url, options = {}) {
+    const fetch = (await import('node-fetch')).default;
+    const controller = new AbortController();
+    const timeout = options.timeout || FETCH_TIMEOUT;
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    try {
+        const response = await fetch(url, {
+            headers: {
+                'User-Agent': USER_AGENT,
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                'Accept-Language': 'en-US,en;q=0.9,es;q=0.8,zh;q=0.7,pt;q=0.6',
+                'Accept-Encoding': 'identity',
+                ...(options.headers || {}),
+            },
+            redirect: 'follow',
+            follow: options.maxRedirects || MAX_REDIRECTS,
+            signal: controller.signal,
+            size: MAX_CONTENT_LENGTH,
+        });
+        clearTimeout(timeoutId);
+        const html = await response.text();
+        const respHeaders = {};
+        response.headers.forEach((v, k) => { respHeaders[k] = v; });
+        return {
+            status: response.status,
+            html,
+            finalUrl: response.url || url,
+            headers: respHeaders,
+        };
+    }
+    catch (error) {
+        clearTimeout(timeoutId);
+        const err = error;
+        if (err.name === 'AbortError')
+            throw new Error(`Fetch timed out after ${timeout}ms: ${url}`);
+        throw new Error(`Fetch failed for ${url}: ${err.message}`);
+    }
+}
+// ─── HTML Parsing Utilities ─────────────────────────────────────────────────
+/**
+ * Extract text content from HTML, stripping tags.
+ * Lightweight — no external DOM parser dependency.
+ */
+function stripHtml(html) {
+    return html
+        // Remove script and style blocks entirely
+        .replace(/<script[\s\S]*?<\/script>/gi, '')
+        .replace(/<style[\s\S]*?<\/style>/gi, '')
+        .replace(/<noscript[\s\S]*?<\/noscript>/gi, '')
+        // Remove HTML comments
+        .replace(/<!--[\s\S]*?-->/g, '')
+        // Convert block elements to newlines
+        .replace(/<\/(p|div|h[1-6]|li|tr|br|blockquote|article|section|header|footer|nav|aside)>/gi, '\n')
+        .replace(/<(br|hr)\s*\/?>/gi, '\n')
+        // Remove remaining tags
+        .replace(/<[^>]+>/g, '')
+        // Decode common HTML entities
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n)))
+        // Clean up whitespace
+        .replace(/[ \t]+/g, ' ')
+        .replace(/\n\s*\n/g, '\n\n')
+        .trim();
+}
+/**
+ * Extract the main content from HTML using a content-density heuristic.
+ * Inspired by Mozilla's Readability algorithm but simplified.
+ * Scores text blocks by density (text-to-tag ratio) and picks the richest zone.
+ */
+function extractMainContent(html) {
+    // Remove nav, header, footer, sidebar, ads
+    let cleaned = html
+        .replace(/<(nav|header|footer|aside|menu|sidebar)[\s\S]*?<\/\1>/gi, '')
+        .replace(/<div[^>]*(sidebar|menu|nav|footer|header|banner|ad|promo|cookie|popup|modal|overlay)[^>]*>[\s\S]*?<\/div>/gi, '')
+        .replace(/<(script|style|noscript|iframe|svg)[\s\S]*?<\/\1>/gi, '')
+        .replace(/<!--[\s\S]*?-->/g, '');
+    // Try to find <article> or <main> first
+    const articleMatch = cleaned.match(/<(article|main)[^>]*>([\s\S]*?)<\/\1>/i);
+    if (articleMatch) {
+        return stripHtml(articleMatch[2]);
+    }
+    // Content-density approach: split into blocks, score each
+    const blocks = cleaned.split(/<(?:div|section|article|p)[^>]*>/i);
+    let bestBlock = '';
+    let bestScore = 0;
+    for (const block of blocks) {
+        const text = stripHtml(block);
+        const words = text.split(/\s+/).filter(w => w.length > 2);
+        const tagCount = (block.match(/<[^>]+>/g) || []).length;
+        // Score = word count penalized by tag density
+        const score = words.length - (tagCount * 2);
+        if (score > bestScore && words.length > 20) {
+            bestScore = score;
+            bestBlock = text;
+        }
+    }
+    // Fallback: just strip the whole thing
+    if (!bestBlock || bestBlock.length < 100) {
+        return stripHtml(cleaned);
+    }
+    return bestBlock;
+}
+/**
+ * Extract metadata from HTML <head>.
+ */
+function extractMetadata(html) {
+    const meta = {};
+    // <title>
+    const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    if (titleMatch)
+        meta.title = stripHtml(titleMatch[1]).trim();
+    // <meta> tags
+    const metaRegex = /<meta\s+(?:[^>]*?(?:name|property)\s*=\s*["']([^"']+)["'][^>]*?content\s*=\s*["']([^"']+)["']|[^>]*?content\s*=\s*["']([^"']+)["'][^>]*?(?:name|property)\s*=\s*["']([^"']+)["'])[^>]*\/?>/gi;
+    let match;
+    while ((match = metaRegex.exec(html)) !== null) {
+        const key = (match[1] || match[4] || '').toLowerCase();
+        const value = match[2] || match[3] || '';
+        if (key && value)
+            meta[key] = value;
+    }
+    return meta;
+}
+/**
+ * Extract all href links from HTML.
+ */
+function extractLinks(html, baseUrl) {
+    const links = new Set();
+    const hrefRegex = /href\s*=\s*["']([^"'#]+)["']/gi;
+    let match;
+    while ((match = hrefRegex.exec(html)) !== null) {
+        let href = match[1].trim();
+        if (!href || href.startsWith('javascript:') || href.startsWith('mailto:') || href.startsWith('tel:'))
+            continue;
+        try {
+            const resolved = new URL(href, baseUrl).href;
+            links.add(resolved);
+        }
+        catch {
+            // Invalid URL, skip
+        }
+    }
+    return Array.from(links);
+}
+/**
+ * Extract all image src URLs from HTML.
+ */
+function extractImages(html, baseUrl) {
+    const images = new Set();
+    const srcRegex = /<img[^>]+src\s*=\s*["']([^"']+)["']/gi;
+    let match;
+    while ((match = srcRegex.exec(html)) !== null) {
+        try {
+            const resolved = new URL(match[1].trim(), baseUrl).href;
+            images.add(resolved);
+        }
+        catch {
+            // Skip invalid
+        }
+    }
+    return Array.from(images);
+}
+/**
+ * Check if a URL is in the same domain as the base.
+ */
+function isSameDomain(url, baseUrl) {
+    try {
+        const a = new URL(url);
+        const b = new URL(baseUrl);
+        return a.hostname === b.hostname;
+    }
+    catch {
+        return false;
+    }
+}
+// ─── 1. SEARCH ENGINE ──────────────────────────────────────────────────────
+/**
+ * Native web search using DuckDuckGo HTML scraping.
+ * No API key needed. Free. Unlimited.
+ *
+ * Fallback order: DuckDuckGo HTML → Google HTML (if DDG blocked) → SearXNG (if configured)
+ */
+export async function nativeSearch(query, options = {}) {
+    const startTime = Date.now();
+    const maxResults = options.max_results || 10;
+    // Try DuckDuckGo first
+    try {
+        const results = await searchDuckDuckGo(query, maxResults, options.region, options.time_range);
+        if (results.length > 0) {
+            return {
+                query,
+                results,
+                total_results: results.length,
+                search_engine: 'duckduckgo',
+                response_time_ms: Date.now() - startTime,
+            };
+        }
+    }
+    catch {
+        // DDG failed, try fallback
+    }
+    // Fallback: SearXNG if configured
+    if (options.searxng_url) {
+        try {
+            const results = await searchSearXNG(query, maxResults, options.searxng_url);
+            if (results.length > 0) {
+                return {
+                    query,
+                    results,
+                    total_results: results.length,
+                    search_engine: 'searxng',
+                    response_time_ms: Date.now() - startTime,
+                };
+            }
+        }
+        catch {
+            // SearXNG failed
+        }
+    }
+    // Fallback: Google HTML scraping
+    try {
+        const results = await searchGoogleHTML(query, maxResults, options.region);
+        return {
+            query,
+            results,
+            total_results: results.length,
+            search_engine: 'google',
+            response_time_ms: Date.now() - startTime,
+        };
+    }
+    catch {
+        // All engines failed
+    }
+    return {
+        query,
+        results: [],
+        total_results: 0,
+        search_engine: 'none',
+        response_time_ms: Date.now() - startTime,
+    };
+}
+async function searchDuckDuckGo(query, maxResults, region, timeRange) {
+    const params = new URLSearchParams({
+        q: query,
+        t: 'h_', // Text-only mode
+        ia: 'web',
+    });
+    if (region)
+        params.set('kl', region);
+    if (timeRange)
+        params.set('df', timeRange);
+    const url = `https://html.duckduckgo.com/html/?${params.toString()}`;
+    const { html } = await safeFetch(url, {
+        headers: { 'Accept': 'text/html' },
+    });
+    const results = [];
+    // Parse DuckDuckGo HTML results
+    // Each result is in a div with class "result"
+    const resultBlocks = html.split(/class="result\s/);
+    for (let i = 1; i < resultBlocks.length && results.length < maxResults; i++) {
+        const block = resultBlocks[i];
+        // Extract URL from result__a href
+        const urlMatch = block.match(/class="result__a"[^>]*href="([^"]+)"/);
+        if (!urlMatch)
+            continue;
+        let resultUrl = urlMatch[1];
+        // DuckDuckGo sometimes wraps URLs in a redirect
+        if (resultUrl.includes('uddg=')) {
+            const decoded = decodeURIComponent(resultUrl.split('uddg=')[1]?.split('&')[0] || '');
+            if (decoded)
+                resultUrl = decoded;
+        }
+        // Extract title
+        const titleMatch = block.match(/class="result__a"[^>]*>([\s\S]*?)<\/a>/);
+        const title = titleMatch ? stripHtml(titleMatch[1]).trim() : '';
+        // Extract snippet
+        const snippetMatch = block.match(/class="result__snippet"[^>]*>([\s\S]*?)<\/(?:a|span|div)/);
+        const snippet = snippetMatch ? stripHtml(snippetMatch[1]).trim() : '';
+        if (resultUrl && title) {
+            results.push({
+                title,
+                url: resultUrl,
+                snippet,
+                source: 'duckduckgo',
+                position: results.length + 1,
+            });
+        }
+    }
+    return results;
+}
+async function searchGoogleHTML(query, maxResults, region) {
+    const params = new URLSearchParams({
+        q: query,
+        num: String(Math.min(maxResults, 20)),
+    });
+    if (region)
+        params.set('hl', region.split('-')[0] || 'en');
+    const url = `https://www.google.com/search?${params.toString()}`;
+    const { html } = await safeFetch(url, {
+        headers: {
+            'Accept': 'text/html',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        },
+    });
+    const results = [];
+    // Google wraps results in <div class="g">
+    const blocks = html.split(/<div class="g"/);
+    for (let i = 1; i < blocks.length && results.length < maxResults; i++) {
+        const block = blocks[i];
+        // URL in <a href="">
+        const urlMatch = block.match(/<a[^>]+href="(https?:\/\/[^"]+)"/);
+        if (!urlMatch)
+            continue;
+        const resultUrl = urlMatch[1];
+        // Title inside <h3>
+        const titleMatch = block.match(/<h3[^>]*>([\s\S]*?)<\/h3>/);
+        const title = titleMatch ? stripHtml(titleMatch[1]).trim() : '';
+        // Snippet in various spans
+        const snippetMatch = block.match(/data-sncf="[^"]*"[^>]*>([\s\S]*?)<\/(?:span|div)>/);
+        const snippet = snippetMatch ? stripHtml(snippetMatch[1]).trim() : '';
+        if (resultUrl && title && !resultUrl.includes('google.com/search')) {
+            results.push({
+                title,
+                url: resultUrl,
+                snippet,
+                source: 'google',
+                position: results.length + 1,
+            });
+        }
+    }
+    return results;
+}
+async function searchSearXNG(query, maxResults, instanceUrl) {
+    const params = new URLSearchParams({
+        q: query,
+        format: 'json',
+        categories: 'general',
+    });
+    const url = `${instanceUrl.replace(/\/$/, '')}/search?${params.toString()}`;
+    const fetch = (await import('node-fetch')).default;
+    const response = await fetch(url, {
+        headers: { 'Accept': 'application/json', 'User-Agent': USER_AGENT },
+        timeout: FETCH_TIMEOUT,
+    });
+    if (!response.ok)
+        throw new Error(`SearXNG returned ${response.status}`);
+    const data = await response.json();
+    const results = [];
+    for (const item of (data.results || []).slice(0, maxResults)) {
+        results.push({
+            title: item.title || '',
+            url: item.url || '',
+            snippet: item.content || '',
+            source: 'searxng',
+            position: results.length + 1,
+        });
+    }
+    return results;
+}
+// ─── 2. CONTENT EXTRACTOR ──────────────────────────────────────────────────
+/**
+ * Extract clean, structured content from one or more URLs.
+ * Native implementation — no Tavily, no Readability.js dependency.
+ * Uses content-density algorithm + metadata extraction.
+ */
+export async function nativeExtract(urls, options = {}) {
+    const startTime = Date.now();
+    const urlArray = Array.isArray(urls) ? urls : [urls];
+    const results = [];
+    const failed = [];
+    // Process URLs concurrently (max 5 at a time)
+    const batchSize = 5;
+    for (let i = 0; i < urlArray.length; i += batchSize) {
+        const batch = urlArray.slice(i, i + batchSize);
+        const batchResults = await Promise.allSettled(batch.map(url => extractSingleUrl(url, options)));
+        for (let j = 0; j < batchResults.length; j++) {
+            const result = batchResults[j];
+            if (result.status === 'fulfilled') {
+                results.push(result.value);
+            }
+            else {
+                failed.push({ url: batch[j], error: result.reason?.message || 'Unknown error' });
+            }
+        }
+    }
+    return {
+        results,
+        failed,
+        response_time_ms: Date.now() - startTime,
+    };
+}
+async function extractSingleUrl(url, options) {
+    const { html, finalUrl } = await safeFetch(url, { timeout: options.timeout });
+    const metadata = extractMetadata(html);
+    const content = extractMainContent(html);
+    const words = content.split(/\s+/).filter(w => w.length > 0);
+    const result = {
+        url: finalUrl,
+        title: metadata.title || metadata['og:title'] || '',
+        description: metadata.description || metadata['og:description'] || '',
+        content,
+        language: metadata['og:locale'] || metadata.language || detectLanguage(content),
+        published_date: metadata['article:published_time'] || metadata['date'] || undefined,
+        author: metadata.author || metadata['article:author'] || undefined,
+        word_count: words.length,
+        links: (options.include_links !== false) ? extractLinks(html, finalUrl) : [],
+        images: options.include_images ? extractImages(html, finalUrl) : [],
+        metadata,
+    };
+    if (options.include_raw_html) {
+        result.raw_html = html;
+    }
+    return result;
+}
+/**
+ * Simple language detection from text content.
+ */
+function detectLanguage(text) {
+    const sample = text.slice(0, 1000).toLowerCase();
+    // Spanish indicators
+    if (/\b(el|la|los|las|que|por|para|como|pero|con|del|una|este|esta)\b/g.test(sample)) {
+        const spanishCount = (sample.match(/\b(el|la|los|las|que|por|para|como|pero|con|del|una)\b/g) || []).length;
+        if (spanishCount > 5)
+            return 'es';
+    }
+    // Chinese characters
+    if (/[\u4e00-\u9fff]/.test(sample))
+        return 'zh';
+    // Portuguese indicators
+    if (/\b(não|são|também|está|isso|você|pelo|pela)\b/g.test(sample))
+        return 'pt';
+    // Default to English
+    return 'en';
+}
+// ─── 3. WEB CRAWLER ────────────────────────────────────────────────────────
+/**
+ * Native BFS web crawler.
+ * Follows links from a starting URL, collecting content from discovered pages.
+ * Respects: domain scope, depth limits, URL patterns, page limits.
+ */
+export async function nativeCrawl(startUrl, options = {}) {
+    const startTime = Date.now();
+    const maxDepth = Math.min(options.max_depth || 2, 5);
+    const maxPages = Math.min(options.max_pages || 10, 50);
+    const sameDomain = options.same_domain_only !== false;
+    const visited = new Set();
+    const pages = [];
+    let totalLinksDiscovered = 0;
+    let maxDepthReached = 0;
+    // Compile include/exclude patterns
+    const includeRegex = options.include_patterns?.map(p => new RegExp(p, 'i')) || [];
+    const excludeRegex = options.exclude_patterns?.map(p => new RegExp(p, 'i')) || [];
+    // BFS queue: [url, depth]
+    const queue = [[startUrl, 0]];
+    visited.add(normalizeUrl(startUrl));
+    while (queue.length > 0 && pages.length < maxPages) {
+        const [url, depth] = queue.shift();
+        if (depth > maxDepth)
+            continue;
+        if (depth > maxDepthReached)
+            maxDepthReached = depth;
+        try {
+            const { html, finalUrl } = await safeFetch(url, { timeout: 10000 });
+            const metadata = extractMetadata(html);
+            const content = extractMainContent(html);
+            const links = extractLinks(html, finalUrl);
+            totalLinksDiscovered += links.length;
+            pages.push({
+                url: finalUrl,
+                title: metadata.title || '',
+                content,
+                depth,
+                links_found: links.length,
+                word_count: content.split(/\s+/).filter(w => w.length > 0).length,
+            });
+            // Enqueue child links
+            if (depth < maxDepth) {
+                for (const link of links) {
+                    const normalized = normalizeUrl(link);
+                    if (visited.has(normalized))
+                        continue;
+                    if (sameDomain && !isSameDomain(link, startUrl))
+                        continue;
+                    if (includeRegex.length > 0 && !includeRegex.some(r => r.test(link)))
+                        continue;
+                    if (excludeRegex.some(r => r.test(link)))
+                        continue;
+                    if (!isValidCrawlUrl(link))
+                        continue;
+                    visited.add(normalized);
+                    queue.push([link, depth + 1]);
+                }
+            }
+        }
+        catch {
+            // Skip pages that fail to fetch
+        }
+    }
+    return {
+        base_url: startUrl,
+        pages,
+        total_pages: pages.length,
+        total_links_discovered: totalLinksDiscovered,
+        max_depth_reached: maxDepthReached,
+        response_time_ms: Date.now() - startTime,
+    };
+}
+function normalizeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        // Remove trailing slash, fragment, common tracking params
+        parsed.hash = '';
+        parsed.searchParams.delete('utm_source');
+        parsed.searchParams.delete('utm_medium');
+        parsed.searchParams.delete('utm_campaign');
+        parsed.searchParams.delete('ref');
+        return parsed.href.replace(/\/+$/, '');
+    }
+    catch {
+        return url;
+    }
+}
+function isValidCrawlUrl(url) {
+    try {
+        const parsed = new URL(url);
+        if (!['http:', 'https:'].includes(parsed.protocol))
+            return false;
+        // Skip common non-content extensions
+        const ext = parsed.pathname.split('.').pop()?.toLowerCase() || '';
+        const skipExts = ['pdf', 'jpg', 'jpeg', 'png', 'gif', 'svg', 'mp4', 'mp3', 'zip', 'tar', 'gz', 'exe', 'dmg', 'css', 'js', 'woff', 'woff2', 'ttf', 'ico'];
+        if (skipExts.includes(ext))
+            return false;
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ─── 4. DEEP RESEARCH ENGINE ───────────────────────────────────────────────
+/**
+ * Multi-step deep research engine.
+ * Performs multiple rounds of: search → extract top results → identify gaps → refine search.
+ *
+ * If a `synthesize` callback is provided (AI model function), it generates a cohesive summary.
+ * Without it, returns structured findings from multiple search rounds.
+ */
+export async function nativeResearch(query, options = {}) {
+    const startTime = Date.now();
+    const maxRounds = options.depth === 'basic' ? 1 : (options.max_rounds || 3);
+    const maxSources = options.max_sources || 15;
+    const allSources = [];
+    const allFindings = [];
+    const seenUrls = new Set();
+    let totalPagesAnalyzed = 0;
+    // Round 1: Initial search
+    let currentQuery = query;
+    for (let round = 0; round < maxRounds && allSources.length < maxSources; round++) {
+        // Search
+        const searchResults = await nativeSearch(currentQuery, {
+            max_results: 8,
+            searxng_url: options.searxng_url,
+        });
+        // Extract content from top results
+        const urlsToExtract = searchResults.results
+            .filter(r => !seenUrls.has(r.url))
+            .slice(0, 5)
+            .map(r => r.url);
+        for (const url of urlsToExtract) {
+            seenUrls.add(url);
+        }
+        if (urlsToExtract.length > 0) {
+            const extracted = await nativeExtract(urlsToExtract, {
+                include_links: false,
+                include_images: false,
+                timeout: 10000,
+            });
+            for (const page of extracted.results) {
+                totalPagesAnalyzed++;
+                // Score relevance by keyword overlap
+                const queryWords = query.toLowerCase().split(/\s+/);
+                const contentLower = page.content.toLowerCase();
+                const matchCount = queryWords.filter(w => contentLower.includes(w)).length;
+                const relevance = matchCount / queryWords.length;
+                allSources.push({
+                    title: page.title,
+                    url: page.url,
+                    snippet: page.content.slice(0, 300),
+                    relevance_score: Math.round(relevance * 100) / 100,
+                });
+                // Extract key sentences as findings
+                const sentences = page.content
+                    .split(/[.!?]+/)
+                    .map(s => s.trim())
+                    .filter(s => s.length > 30 && s.length < 500);
+                const relevantSentences = sentences.filter(s => {
+                    const sLower = s.toLowerCase();
+                    return queryWords.some(w => sLower.includes(w));
+                });
+                allFindings.push(...relevantSentences.slice(0, 3));
+            }
+        }
+        // Refine query for next round based on what we found
+        if (round < maxRounds - 1) {
+            // Extract frequent terms from findings to refine the search
+            const findingsText = allFindings.join(' ').toLowerCase();
+            const wordFreq = new Map();
+            const stopWords = new Set(['the', 'is', 'at', 'which', 'on', 'a', 'an', 'and', 'or', 'but', 'in', 'with', 'to', 'for', 'of', 'that', 'this', 'it', 'be', 'as', 'by', 'from', 'are', 'was', 'were', 'been', 'has', 'have', 'had', 'not', 'can', 'will']);
+            for (const word of findingsText.split(/\s+/)) {
+                if (word.length > 4 && !stopWords.has(word)) {
+                    wordFreq.set(word, (wordFreq.get(word) || 0) + 1);
+                }
+            }
+            // Add top emerging terms to refine the query
+            const topTerms = Array.from(wordFreq.entries())
+                .sort((a, b) => b[1] - a[1])
+                .slice(0, 3)
+                .map(([word]) => word);
+            currentQuery = `${query} ${topTerms.join(' ')}`;
+        }
+    }
+    // Sort sources by relevance
+    allSources.sort((a, b) => b.relevance_score - a.relevance_score);
+    // Deduplicate findings
+    const uniqueFindings = [...new Set(allFindings)].slice(0, 20);
+    // Synthesize if AI callback provided
+    let summary = '';
+    if (options.synthesize && uniqueFindings.length > 0) {
+        try {
+            summary = await options.synthesize(uniqueFindings, query);
+        }
+        catch {
+            summary = `Research on "${query}" found ${allSources.length} sources with ${uniqueFindings.length} key findings across ${maxRounds} search rounds.`;
+        }
+    }
+    else {
+        summary = uniqueFindings.slice(0, 5).join('. ');
+    }
+    return {
+        query,
+        summary,
+        key_findings: uniqueFindings,
+        sources: allSources.slice(0, maxSources),
+        search_rounds: maxRounds,
+        total_pages_analyzed: totalPagesAnalyzed,
+        response_time_ms: Date.now() - startTime,
+    };
+}
+// ─── 5. SITE MAPPER ────────────────────────────────────────────────────────
+/**
+ * Map a website's complete URL structure.
+ * First tries sitemap.xml, then falls back to link-discovery crawl (URL-only mode).
+ * Fast — doesn't extract content, just discovers URLs.
+ */
+export async function nativeMap(url, options = {}) {
+    const startTime = Date.now();
+    const maxUrls = Math.min(options.max_urls || 100, 500);
+    const discoveredUrls = new Set();
+    let sitemapFound = false;
+    // Step 1: Try sitemap.xml
+    try {
+        const sitemapUrls = await parseSitemap(url, maxUrls);
+        if (sitemapUrls.length > 0) {
+            sitemapFound = true;
+            for (const u of sitemapUrls) {
+                discoveredUrls.add(u);
+                if (discoveredUrls.size >= maxUrls)
+                    break;
+            }
+        }
+    }
+    catch {
+        // No sitemap, continue to crawl
+    }
+    // Step 2: If sitemap insufficient, crawl for more URLs
+    if (discoveredUrls.size < maxUrls) {
+        const remaining = maxUrls - discoveredUrls.size;
+        const crawlResult = await nativeCrawl(url, {
+            max_depth: options.max_depth || 3,
+            max_pages: Math.min(remaining, 30), // Crawl max 30 pages for URL discovery
+            same_domain_only: !options.include_subdomains,
+        });
+        for (const page of crawlResult.pages) {
+            discoveredUrls.add(page.url);
+        }
+    }
+    const urlArray = Array.from(discoveredUrls).slice(0, maxUrls);
+    return {
+        base_url: url,
+        urls: urlArray,
+        total_urls: urlArray.length,
+        sitemap_found: sitemapFound,
+        response_time_ms: Date.now() - startTime,
+    };
+}
+async function parseSitemap(baseUrl, maxUrls) {
+    const urls = [];
+    // Try common sitemap locations
+    const sitemapPaths = ['/sitemap.xml', '/sitemap_index.xml', '/sitemap/sitemap.xml'];
+    for (const path of sitemapPaths) {
+        try {
+            const sitemapUrl = new URL(path, baseUrl).href;
+            const { html: xml } = await safeFetch(sitemapUrl, { timeout: 10000 });
+            if (!xml.includes('<urlset') && !xml.includes('<sitemapindex'))
+                continue;
+            // Check if it's a sitemap index (contains other sitemaps)
+            if (xml.includes('<sitemapindex')) {
+                const sitemapLocs = xml.match(/<loc>(.*?)<\/loc>/gi) || [];
+                for (const loc of sitemapLocs.slice(0, 5)) {
+                    const childUrl = loc.replace(/<\/?loc>/gi, '').trim();
+                    try {
+                        const { html: childXml } = await safeFetch(childUrl, { timeout: 10000 });
+                        const childLocs = childXml.match(/<loc>(.*?)<\/loc>/gi) || [];
+                        for (const childLoc of childLocs) {
+                            const pageUrl = childLoc.replace(/<\/?loc>/gi, '').trim();
+                            urls.push(pageUrl);
+                            if (urls.length >= maxUrls)
+                                return urls;
+                        }
+                    }
+                    catch {
+                        continue;
+                    }
+                }
+            }
+            else {
+                // Regular sitemap with <url> entries
+                const locs = xml.match(/<loc>(.*?)<\/loc>/gi) || [];
+                for (const loc of locs) {
+                    const pageUrl = loc.replace(/<\/?loc>/gi, '').trim();
+                    urls.push(pageUrl);
+                    if (urls.length >= maxUrls)
+                        return urls;
+                }
+            }
+            if (urls.length > 0)
+                break; // Found valid sitemap, stop trying others
+        }
+        catch {
+            continue;
+        }
+    }
+    // Also try robots.txt for sitemap references
+    if (urls.length === 0) {
+        try {
+            const robotsUrl = new URL('/robots.txt', baseUrl).href;
+            const { html: robotsTxt } = await safeFetch(robotsUrl, { timeout: 5000 });
+            const sitemapMatches = robotsTxt.match(/Sitemap:\s*(.+)/gi) || [];
+            for (const match of sitemapMatches) {
+                const sitemapUrl = match.replace(/Sitemap:\s*/i, '').trim();
+                try {
+                    const childUrls = await parseSitemap(sitemapUrl, maxUrls - urls.length);
+                    urls.push(...childUrls);
+                    if (urls.length >= maxUrls)
+                        return urls;
+                }
+                catch {
+                    continue;
+                }
+            }
+        }
+        catch {
+            // No robots.txt
+        }
+    }
+    return urls;
+}
+//# sourceMappingURL=web-engine.js.map