webpeel 0.20.3 → 0.20.5

package/dist/cli/commands/fetch.js

@@ -13,6 +13,14 @@ import { parseActions, formatError, fetchViaApi, outputResult, writeStdout, buil
 // ─── runFetch ─────────────────────────────────────────────────────────────────
 // Main fetch handler — shared with the `pipe` and `ask` subcommands
 export async function runFetch(url, options) {
+    // --content-only: override all output flags — we just want raw content
+    if (options.contentOnly) {
+        options.silent = true;
+        // Disable json/text/html — we output content directly
+        options.json = false;
+        options.html = false;
+        options.text = false;
+    }
     // Handle --format flag: maps to existing boolean flags
     if (options.format) {
         const fmt = options.format.toLowerCase();
@@ -30,9 +38,10 @@ export async function runFetch(url, options) {
     }
     // Smart defaults: when piped (not a TTY), default to silent JSON + budget
     // BUT respect explicit --format flag (user chose the output format)
+    // AND respect --content-only (raw content output, no JSON wrapper)
     const isPiped = !process.stdout.isTTY;
     const hasExplicitFormat = options.format && ['text', 'html', 'markdown', 'md'].includes(options.format.toLowerCase());
-    if (isPiped && !options.html && !options.text && !hasExplicitFormat) {
+    if (isPiped && !options.html && !options.text && !hasExplicitFormat && !options.contentOnly) {
         if (!options.json)
             options.json = true;
         if (!options.silent)
@@ -284,11 +293,38 @@ export async function runFetch(url, options) {
                     cachedResult.extracted = extractedCached;
                 }
             }
-            await outputResult(cachedResult, options, { cached: true });
+            if (options.contentOnly) {
+                await writeStdout(cachedResult.content + '\n');
+            }
+            else {
+                await outputResult(cachedResult, options, { cached: true });
+            }
             process.exit(0);
         }
     }
-    const spinner = options.silent ? null : ora('Fetching...').start();
+    // --progress: show escalation steps on stderr (overrides spinner)
+    let progressInterval;
+    const progressStart = Date.now();
+    if (options.progress) {
+        process.stderr.write(`[simple] Fetching ${url}...\n`);
+        // Show escalation hints based on elapsed time (best-effort approximations)
+        const progressSteps = [
+            { afterMs: 2500, message: '[simple] Waiting for response...' },
+            { afterMs: 6000, message: '[browser] Simple too slow — escalating to browser render...' },
+            { afterMs: 12000, message: '[browser] Rendering with Chromium...' },
+            { afterMs: 20000, message: '[stealth] Escalating to stealth mode...' },
+        ];
+        let stepIdx = 0;
+        progressInterval = setInterval(() => {
+            const elapsed = Date.now() - progressStart;
+            while (stepIdx < progressSteps.length && elapsed >= progressSteps[stepIdx].afterMs) {
+                process.stderr.write(`${progressSteps[stepIdx].message}\n`);
+                stepIdx++;
+            }
+        }, 500);
+    }
+    // Suppress spinner when --progress is active (progress lines replace it)
+    const spinner = (options.silent || options.progress) ? null : ora('Fetching...').start();
     try {
         // Validate options
         if (options.wait && (options.wait < 0 || options.wait > 60000)) {
@@ -528,7 +564,22 @@ export async function runFetch(url, options) {
         if (resolvedProfileName) {
             touchProfile(resolvedProfileName);
         }
-        if (spinner) {
+        // Stop progress interval and show final result
+        if (progressInterval) {
+            clearInterval(progressInterval);
+            progressInterval = undefined;
+        }
+        if (options.progress) {
+            const method = result.method || 'simple';
+            const elapsedSec = ((result.elapsed || (Date.now() - progressStart)) / 1000).toFixed(1);
+            const tokenCount = (result.tokens || 0).toLocaleString();
+            // Show escalation arrow if browser/stealth was needed
+            if (method !== 'simple') {
+                process.stderr.write(`[simple] → [${method}] escalated\n`);
+            }
+            process.stderr.write(`[${method}] Done — ${tokenCount} tokens in ${elapsedSec}s\n`);
+        }
+        else if (spinner) {
             const domainTag = result.domainData
                 ? ` [${result.domainData.domain}:${result.domainData.type}]`
                 : '';
@@ -866,11 +917,27 @@ export async function runFetch(url, options) {
                     result.extracted = extracted;
                 }
             }
-            // Output results (default path)
-            await outputResult(result, options, {
-                cached: false,
-                truncated: contentTruncated || undefined,
-            });
+            // --content-only: output raw content only, no wrapper
+            if (options.contentOnly) {
+                await writeStdout(result.content + '\n');
+            }
+            else {
+                // Output results (default path)
+                await outputResult(result, options, {
+                    cached: false,
+                    truncated: contentTruncated || undefined,
+                });
+                // Token savings display (our unique selling point)
+                if (!options.json && !options.silent && result.tokenSavingsPercent) {
+                    const savings = result.tokenSavingsPercent;
+                    const raw = result.rawTokenEstimate;
+                    const optimized = result.tokens || 0;
+                    if (savings > 0) {
+                        const rawStr = raw ? `${raw.toLocaleString()}→${optimized.toLocaleString()} tokens` : `${optimized.toLocaleString()} tokens`;
+                        process.stderr.write(`\x1b[32m💰 Token savings: ${savings}% smaller than raw HTML (${rawStr})\x1b[0m\n`);
+                    }
+                }
+            }
         }
         // Clean up and exit
         await cleanup();
@@ -976,6 +1043,8 @@ export function registerFetchCommands(program) {
         .option('--wait-selector <css>', 'Wait for CSS selector before extracting (auto-enables --render)')
         .option('--block-resources <types>', 'Block resource types, comma-separated: image,stylesheet,font,media,script (auto-enables --render)')
         .option('--format <type>', 'Output format: markdown (default), text, html, json')
+        .option('--content-only', 'Output only the raw content field (no metadata, no JSON wrapper) — ideal for piping to LLMs')
+        .option('--progress', 'Show engine escalation steps (simple → browser → stealth) with timing')
         .action(async (url, options) => {
         await runFetch(url, options);
     });

package/dist/cli/commands/search.js

@@ -23,6 +23,7 @@ export function registerSearchCommands(program) {
         .option('--budget <n>', 'Token budget for site-search result content', parseInt)
         .option('-s, --silent', 'Silent mode')
         .option('--proxy <url>', 'Proxy URL for requests (http://host:port, socks5://user:pass@host:port)')
+        .option('--fetch', 'Also fetch and include content from each result URL')
         .option('--agent', 'Agent mode: sets --json, --silent, and --budget 4000 (override with --budget N)')
         .action(async (query, options) => {
         // --agent sets sensible defaults for AI agents; explicit flags override
@@ -178,9 +179,61 @@ export function registerSearchCommands(program) {
             const searchData = await searchRes.json();
             // API returns { success: true, data: { web: [...] } } or { results: [...] }
             let results = searchData.data?.web || searchData.data?.results || searchData.results || [];
+            // Client-side ad filtering: remove DuckDuckGo ads that slip through the server
+            results = results.filter(r => {
+                // Filter DDG-internal URLs
+                try {
+                    const parsed = new URL(r.url);
+                    if (parsed.hostname === 'duckduckgo.com')
+                        return false;
+                    if (parsed.searchParams.has('ad_domain') ||
+                        parsed.searchParams.has('ad_provider') ||
+                        parsed.searchParams.has('ad_type'))
+                        return false;
+                }
+                catch {
+                    return false;
+                }
+                // Filter ad snippets
+                if (r.snippet && (r.snippet.includes('Ad ·') ||
+                    r.snippet.includes('Ad Viewing ads is privacy protected by DuckDuckGo') ||
+                    r.snippet.toLowerCase().startsWith('ad ·')))
+                    return false;
+                return true;
+            });
             if (spinner) {
                 spinner.succeed(`Found ${results.length} results`);
             }
+            // --fetch: fetch content from each result
+            if (options.fetch && results.length > 0) {
+                const fetchCfg = loadConfig();
+                const fetchApiKey = fetchCfg.apiKey || process.env.WEBPEEL_API_KEY;
+                const fetchApiUrl = process.env.WEBPEEL_API_URL || 'https://api.webpeel.dev';
+                if (fetchApiKey) {
+                    const fetchSpinner = isSilent ? null : ora(`Fetching content from ${results.length} results...`).start();
+                    await Promise.all(results.map(async (result) => {
+                        try {
+                            const fetchParams = new URLSearchParams({ url: result.url });
+                            if (options.budget)
+                                fetchParams.set('budget', String(options.budget || 2000));
+                            const fetchRes = await fetch(`${fetchApiUrl}/v1/fetch?${fetchParams}`, {
+                                headers: { Authorization: `Bearer ${fetchApiKey}` },
+                                signal: AbortSignal.timeout(20000),
+                            });
+                            if (fetchRes.ok) {
+                                const fetchData = await fetchRes.json();
+                                result.content = fetchData.content || fetchData.data?.content || '';
+                            }
+                        }
+                        catch { /* skip on error */ }
+                    }));
+                    if (fetchSpinner)
+                        fetchSpinner.succeed('Content fetched');
+                }
+                else if (!isSilent) {
+                    console.error('Warning: --fetch requires API key (run: webpeel auth <key>)');
+                }
+            }
             // Show usage footer for free/anonymous users
             if (usageCheck.usageInfo && !isSilent) {
                 showUsageFooter(usageCheck.usageInfo, usageCheck.isAnonymous || false, false);
@@ -196,10 +249,24 @@ export function registerSearchCommands(program) {
                 await writeStdout(jsonStr + '\n');
             }
             else {
-                for (const result of results) {
-                    console.log(`\n${result.title}`);
-                    console.log(result.url);
-                    console.log(result.snippet);
+                // Human-readable numbered results
+                if (results.length === 0) {
+                    await writeStdout('No results found.\n');
+                }
+                else {
+                    await writeStdout(`\n`);
+                    for (const [i, result] of results.entries()) {
+                        await writeStdout(`${i + 1}. ${result.title}\n`);
+                        await writeStdout(`   ${result.url}\n`);
+                        if (result.snippet) {
+                            await writeStdout(`   ${result.snippet}\n`);
+                        }
+                        if (result.content) {
+                            const preview = result.content.slice(0, 500);
+                            await writeStdout(`\n   --- Content ---\n${preview}${result.content.length > 500 ? '\n   [...]' : ''}\n`);
+                        }
+                        await writeStdout('\n');
+                    }
                 }
             }
             process.exit(0);

package/dist/cli/utils.js

@@ -508,13 +508,11 @@ export async function outputResult(result, options, extra = {}) {
     // Default: full output
     if (options.json) {
         // Build clean JSON output with guaranteed top-level fields
+        // Note: elapsed/method/tokens are placed at the END so `tail -3` shows perf metrics
         const output = {
             url: result.url,
             title: result.metadata?.title || result.title || null,
-            tokens: result.tokens || 0,
             fetchedAt: new Date().toISOString(),
-            method: result.method || 'simple',
-            elapsed: result.elapsed,
             content: result.content,
         };
         // Add optional fields only if present (filter out undefined/null values from metadata)
@@ -529,6 +527,10 @@ export async function outputResult(result, options, extra = {}) {
         }
         if (result.links?.length)
             output.links = result.links;
+        if (result.tokenSavingsPercent !== undefined)
+            output.tokenSavingsPercent = result.tokenSavingsPercent;
+        if (result.rawTokenEstimate !== undefined)
+            output.rawTokenEstimate = result.rawTokenEstimate;
         if (result.images?.length)
             output.images = result.images;
         if (result.structured)
@@ -562,6 +564,10 @@ export async function outputResult(result, options, extra = {}) {
         if (extra.totalAvailable !== undefined)
             output.totalAvailable = extra.totalAvailable;
         output._meta = { version: cliVersion, method: result.method || 'simple', timing: result.timing, serverMarkdown: result.serverMarkdown || false };
+        // Perf metrics at the end — `tail -3` shows: elapsed | method | tokens
+        output.elapsed = result.elapsed;
+        output.method = result.method || 'simple';
+        output.tokens = result.tokens || 0;
         await writeStdout(JSON.stringify(output, null, 2) + '\n');
     }
     else {
@@ -586,10 +592,11 @@ export async function outputResult(result, options, extra = {}) {
         }
         // Stream content immediately to stdout — consumer gets it without waiting
         await writeStdout(result.content + '\n');
-        // Append timing summary to stderr so it doesn't pollute piped content
-        if (!options.silent) {
+        // Append timing summary to stderr (always — doesn't pollute stdout pipe)
+        {
             const totalMs = result.timing?.total ?? result.elapsed;
-            process.stderr.write(`\n--- ${result.tokens} tokens · ${totalMs}ms ---\n`);
+            const method = result.method || 'simple';
+            process.stderr.write(`\n--- ${totalMs}ms | ${method} | ${result.tokens} tokens ---\n`);
         }
     }
 }

package/dist/core/image-caption.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Image alt-text enhancement module.
+ *
+ * Two strategies:
+ * 1. Heuristic (no LLM) — generates captions from filename, URL path, and nearby text context
+ * 2. LLM vision (with user's API key) — generates accurate descriptions via vision models
+ */
+/**
+ * Enhance images that lack alt text with heuristic-based descriptions.
+ *
+ * Processes <img> tags that have:
+ * - No alt attribute at all
+ * - An empty alt attribute (alt="")
+ *
+ * Caption priority:
+ * 1. Filename analysis: `/images/team-photo-2024.jpg` → "Team Photo 2024"
+ * 2. URL path segments: `/products/widget/hero.png` → "Widget image"
+ * 3. Nearby heading/figcaption/paragraph text (within 300 chars)
+ * 4. Generic fallback: "Image"
+ *
+ * Non-empty alt text is always preserved unchanged.
+ *
+ * @param html - Raw HTML string to process
+ * @returns HTML with alt text added/replaced on qualifying img tags
+ */
+export declare function enhanceImageAltText(html: string): string;
+/**
+ * Caption images using LLM vision models.
+ *
+ * Requires the user to supply their own API key. No key is stored server-side.
+ * Processes images sequentially to avoid rate limiting.
+ *
+ * @param images - Array of {url, context} pairs. `context` is nearby text for better accuracy.
+ * @param llmApiKey - API key for the chosen provider
+ * @param llmProvider - Vision-capable model to use: 'openai' | 'anthropic' | 'google'
+ * @returns Array of {url, caption} — same order as input
+ */
+export declare function captionImagesWithLLM(images: {
+    url: string;
+    context: string;
+}[], llmApiKey: string, llmProvider: 'openai' | 'anthropic' | 'google'): Promise<{
+    url: string;
+    caption: string;
+}[]>;

package/dist/core/image-caption.js

@@ -0,0 +1,271 @@
+/**
+ * Image alt-text enhancement module.
+ *
+ * Two strategies:
+ * 1. Heuristic (no LLM) — generates captions from filename, URL path, and nearby text context
+ * 2. LLM vision (with user's API key) — generates accurate descriptions via vision models
+ */
+// ---------------------------------------------------------------------------
+// Heuristic helpers
+// ---------------------------------------------------------------------------
+const GENERIC_FILENAMES = new Set([
+    'image', 'img', 'photo', 'picture', 'thumbnail', 'thumb',
+    'icon', 'logo', 'banner', 'placeholder', 'default', 'hero',
+    'bg', 'background', 'avatar', 'pic', 'graphic', 'figure', 'shot',
+]);
+const NOISE_PATH_SEGMENTS = new Set([
+    'images', 'img', 'imgs', 'photos', 'assets', 'static', 'media',
+    'public', 'uploads', 'files', 'resources', 'content', 'cdn',
+    'dist', 'build', 'src', 'www', 'web', 'site',
+]);
+/**
+ * Convert a URL slug / camelCase / underscored name into readable title-cased text.
+ * Examples:
+ *   "team-photo-2024" → "Team Photo 2024"
+ *   "heroImage"       → "Hero Image"
+ *   "my_product_shot" → "My Product Shot"
+ */
+function slugToTitle(slug) {
+    return slug
+        .replace(/[-_]+/g, ' ') // hyphens/underscores → spaces
+        .replace(/([a-z])([A-Z])/g, '$1 $2') // camelCase split
+        .replace(/([A-Z]{2,})([A-Z][a-z])/g, '$1 $2') // HTMLParser → HTML Parser
+        .replace(/\s+/g, ' ')
+        .trim()
+        .replace(/\b\w/g, (c) => c.toUpperCase()); // Title Case
+}
+/**
+ * Derive a caption from the image src URL.
+ * Tries (in order): filename, parent path segment.
+ * Returns null if nothing useful can be derived.
+ */
+function captionFromUrl(src) {
+    try {
+        const pathStr = src.startsWith('http') ? new URL(src).pathname : src;
+        const parts = pathStr.split('/').filter(Boolean);
+        // 1. Try the filename (without extension)
+        const filename = parts[parts.length - 1] ?? '';
+        const nameWithoutExt = filename.replace(/\.[^.]+$/, '');
+        if (nameWithoutExt.length > 2 &&
+            !GENERIC_FILENAMES.has(nameWithoutExt.toLowerCase())) {
+            const title = slugToTitle(nameWithoutExt);
+            if (title.length > 2)
+                return title;
+        }
+        // 2. Try meaningful parent path segments (walk up from the file)
+        for (let i = parts.length - 2; i >= 0; i--) {
+            const seg = parts[i];
+            if (seg && seg.length > 2 && !NOISE_PATH_SEGMENTS.has(seg.toLowerCase())) {
+                const title = slugToTitle(seg);
+                return `${title} image`;
+            }
+        }
+    }
+    catch {
+        // URL parse error — fall through
+    }
+    return null;
+}
+/**
+ * Extract the nearest meaningful text context surrounding an img tag.
+ * Searches up to 300 chars before and after the tag position.
+ * Prefers headings, then figcaption, then raw surrounding text.
+ */
+function extractNearbyText(html, imgStart) {
+    const beforeHtml = html.slice(Math.max(0, imgStart - 300), imgStart);
+    const afterHtml = html.slice(imgStart, Math.min(html.length, imgStart + 400));
+    // Prefer the nearest heading before the image
+    const headingMatches = beforeHtml.match(/<h[1-6][^>]*>([^<]{3,80})<\/h[1-6]>/gi);
+    if (headingMatches) {
+        const lastHeading = headingMatches[headingMatches.length - 1];
+        const text = lastHeading.replace(/<[^>]+>/g, '').trim();
+        if (text.length > 3)
+            return text;
+    }
+    // Prefer figcaption near the image
+    const figMatch = afterHtml.match(/<figcaption[^>]*>([^<]{3,120})<\/figcaption>/i);
+    if (figMatch) {
+        const text = (figMatch[1] ?? '').trim();
+        if (text.length > 3)
+            return text;
+    }
+    // Strip tags, return the richer side
+    const stripTags = (s) => s.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+    const beforeText = stripTags(beforeHtml);
+    const afterText = stripTags(afterHtml);
+    return afterText.length > beforeText.length
+        ? afterText.slice(0, 80)
+        : beforeText.slice(-80);
+}
+// ---------------------------------------------------------------------------
+// Public: heuristic alt-text enhancement
+// ---------------------------------------------------------------------------
+/**
+ * Enhance images that lack alt text with heuristic-based descriptions.
+ *
+ * Processes <img> tags that have:
+ * - No alt attribute at all
+ * - An empty alt attribute (alt="")
+ *
+ * Caption priority:
+ * 1. Filename analysis: `/images/team-photo-2024.jpg` → "Team Photo 2024"
+ * 2. URL path segments: `/products/widget/hero.png` → "Widget image"
+ * 3. Nearby heading/figcaption/paragraph text (within 300 chars)
+ * 4. Generic fallback: "Image"
+ *
+ * Non-empty alt text is always preserved unchanged.
+ *
+ * @param html - Raw HTML string to process
+ * @returns HTML with alt text added/replaced on qualifying img tags
+ */
+export function enhanceImageAltText(html) {
+    return html.replace(/<img(\s[^>]*)>/gi, (match, attrs, offset) => {
+        const srcMatch = attrs.match(/\bsrc=["']([^"']*)["']/i);
+        if (!srcMatch)
+            return match; // No src — leave unchanged
+        const src = srcMatch[1] ?? '';
+        const altMatch = attrs.match(/\balt=["']([^"']*)["']/i);
+        const altValue = altMatch ? altMatch[1] : null;
+        // Already has meaningful alt text — preserve as-is
+        if (altValue !== null && altValue.trim() !== '')
+            return match;
+        // Build caption: URL → nearby text → generic fallback
+        let caption = captionFromUrl(src);
+        if (!caption) {
+            const nearbyText = extractNearbyText(html, offset).trim();
+            if (nearbyText.length > 3) {
+                caption = `Image: ${nearbyText.slice(0, 60)}`;
+            }
+            else {
+                caption = 'Image';
+            }
+        }
+        const escaped = caption.replace(/"/g, '&quot;');
+        if (altMatch) {
+            // Replace the empty alt value in-place
+            const newAttrs = attrs.replace(/\balt=["'][^"']*["']/i, `alt="${escaped}"`);
+            return `<img${newAttrs}>`;
+        }
+        else {
+            // Prepend alt attribute (keeps src first is fine; alt first is valid too)
+            return `<img alt="${escaped}"${attrs}>`;
+        }
+    });
+}
+// ---------------------------------------------------------------------------
+// Public: LLM vision captioning (BYOK)
+// ---------------------------------------------------------------------------
+/**
+ * Caption images using LLM vision models.
+ *
+ * Requires the user to supply their own API key. No key is stored server-side.
+ * Processes images sequentially to avoid rate limiting.
+ *
+ * @param images - Array of {url, context} pairs. `context` is nearby text for better accuracy.
+ * @param llmApiKey - API key for the chosen provider
+ * @param llmProvider - Vision-capable model to use: 'openai' | 'anthropic' | 'google'
+ * @returns Array of {url, caption} — same order as input
+ */
+export async function captionImagesWithLLM(images, llmApiKey, llmProvider) {
+    const results = [];
+    for (const image of images) {
+        try {
+            const prompt = `Write a concise, descriptive alt text (1–2 sentences) for this image. Context from the surrounding page: "${image.context || 'none'}". Be specific and informative.`;
+            let caption = '';
+            if (llmProvider === 'openai') {
+                // GPT-4o-mini supports image_url with public URLs
+                const response = await fetch('https://api.openai.com/v1/chat/completions', {
+                    method: 'POST',
+                    headers: {
+                        Authorization: `Bearer ${llmApiKey}`,
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        model: 'gpt-4o-mini',
+                        max_tokens: 120,
+                        messages: [
+                            {
+                                role: 'user',
+                                content: [
+                                    { type: 'image_url', image_url: { url: image.url, detail: 'low' } },
+                                    { type: 'text', text: prompt },
+                                ],
+                            },
+                        ],
+                    }),
+                });
+                const data = (await response.json());
+                caption = (data?.choices?.[0]?.message?.content ?? '').trim();
+            }
+            else if (llmProvider === 'anthropic') {
+                // claude-haiku-4-5 supports url-type image sources
+                const response = await fetch('https://api.anthropic.com/v1/messages', {
+                    method: 'POST',
+                    headers: {
+                        'x-api-key': llmApiKey,
+                        'anthropic-version': '2023-06-01',
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        model: 'claude-haiku-4-5',
+                        max_tokens: 120,
+                        messages: [
+                            {
+                                role: 'user',
+                                content: [
+                                    { type: 'image', source: { type: 'url', url: image.url } },
+                                    { type: 'text', text: prompt },
+                                ],
+                            },
+                        ],
+                    }),
+                });
+                const data = (await response.json());
+                caption = (data?.content?.[0]?.text ?? '').trim();
+            }
+            else if (llmProvider === 'google') {
+                // Gemini requires base64 inlineData — fetch the image first
+                let imageData = null;
+                let mimeType = 'image/jpeg';
+                try {
+                    const imgResp = await fetch(image.url, {
+                        headers: { Accept: 'image/*,*/*;q=0.8' },
+                    });
+                    if (imgResp.ok) {
+                        const buffer = await imgResp.arrayBuffer();
+                        imageData = Buffer.from(buffer).toString('base64');
+                        const ct = imgResp.headers.get('content-type') ?? 'image/jpeg';
+                        mimeType = ct.split(';')[0]?.trim() ?? 'image/jpeg';
+                    }
+                }
+                catch {
+                    // Image download failed — skip this provider for this image
+                }
+                if (imageData) {
+                    const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-lite:generateContent?key=${llmApiKey}`, {
+                        method: 'POST',
+                        headers: { 'Content-Type': 'application/json' },
+                        body: JSON.stringify({
+                            contents: [
+                                {
+                                    parts: [
+                                        { inlineData: { mimeType, data: imageData } },
+                                        { text: prompt },
+                                    ],
+                                },
+                            ],
+                        }),
+                    });
+                    const data = (await response.json());
+                    caption = (data?.candidates?.[0]?.content?.parts?.[0]?.text ?? '').trim();
+                }
+            }
+            results.push({ url: image.url, caption: caption || 'Image' });
+        }
+        catch {
+            // Non-fatal — captioning failed for this image
+            results.push({ url: image.url, caption: 'Image' });
+        }
+    }
+    return results;
+}

package/dist/core/links.d.ts

@@ -0,0 +1,10 @@
+export interface ExtractedLink {
+    url: string;
+    text: string;
+}
+/**
+ * Extract all links from an HTML string.
+ * Returns a deduplicated list of { url, text } pairs, excluding anchors,
+ * javascript: hrefs, mailto:, and tel: links.
+ */
+export declare function extractLinks(html: string, baseUrl?: string): ExtractedLink[];

package/dist/core/links.js

@@ -0,0 +1,44 @@
+/**
+ * Link extraction from HTML
+ * Extracts all <a href="..."> tags and returns deduplicated { url, text } pairs
+ */
+import { load } from 'cheerio';
+/**
+ * Extract all links from an HTML string.
+ * Returns a deduplicated list of { url, text } pairs, excluding anchors,
+ * javascript: hrefs, mailto:, and tel: links.
+ */
+export function extractLinks(html, baseUrl) {
+    if (!html)
+        return [];
+    const $ = load(html);
+    const seen = new Set();
+    const links = [];
+    $('a[href]').each((_i, el) => {
+        const href = $(el).attr('href') || '';
+        if (!href ||
+            href.startsWith('#') ||
+            href.startsWith('javascript:') ||
+            href.startsWith('mailto:') ||
+            href.startsWith('tel:') ||
+            href.startsWith('data:')) {
+            return;
+        }
+        let url = href;
+        // Resolve relative URLs when baseUrl is provided
+        if (baseUrl && !href.match(/^https?:\/\//)) {
+            try {
+                url = new URL(href, baseUrl).href;
+            }
+            catch {
+                return; // Skip unresolvable relative URLs
+            }
+        }
+        if (!seen.has(url)) {
+            seen.add(url);
+            const text = $(el).text().trim().replace(/\s+/g, ' ');
+            links.push({ url, text });
+        }
+    });
+    return links;
+}

package/dist/core/pipeline.js

@@ -441,6 +441,14 @@ export async function parseContent(ctx) {
     const fetchResult = ctx.fetchResult;
     const { contentType, format, fullPage, raw, selector, exclude, includeTags, excludeTags } = ctx;
     const hasBuffer = !!fetchResult.buffer;
+    // === Image alt-text enhancement (opt-in, heuristic) ===
+    // Runs before any conversion so both lite mode and standard mode benefit.
+    if (ctx.options.captionImages && contentType === 'html' && fetchResult.html) {
+        ctx.timer.mark('captionImages');
+        const { enhanceImageAltText } = await import('./image-caption.js');
+        fetchResult.html = enhanceImageAltText(fetchResult.html);
+        ctx.timer.end('captionImages');
+    }
     if (contentType === 'document' && hasBuffer) {
         // Document parsing pipeline (PDF/DOCX)
         // 'clean' maps to 'markdown' for extraction; cleanForAI is applied in buildResult