sanook-cli 0.5.2 → 0.5.7

package/dist/web-fetch.js

@@ -0,0 +1,637 @@
+import { BRAND } from './brand.js';
+// Honest, identifying User-Agent. Sanook does NOT impersonate a human browser to evade detection.
+export const SANOOK_USER_AGENT = `${BRAND.cliName}-cli (+web-fetch; respects robots.txt)`;
+// Techniques Sanook will NEVER attempt. Surfaced on every fetch result for transparency.
+// (See second-brain/Shared/Tech-Standards/web-search-grounding-policy.md)
+export const REFUSED_TECHNIQUES = [
+    'CAPTCHA solving',
+    'login / authentication bypass',
+    'paywall / metered-content circumvention',
+    'WAF / bot-challenge (Cloudflare etc.) defeat',
+    'browser-fingerprint / TLS spoofing',
+    'residential-proxy rotation to evade blocks',
+];
+// Ordered, cheapest + most-polite first. Documents the ladder for `web status` and docs.
+export const WEB_FETCH_LADDER = [
+    { tier: 0, name: 'preflight', solves: 'SSRF guard + robots.txt + honest UA + timeout/size caps (cross-cutting, always on)' },
+    { tier: 1, name: 'direct', solves: 'normal public HTML — extract title/meta/headings/links/JSON-LD to understand the site' },
+    { tier: 2, name: 'reader', solves: 'messy or JS-shell HTML — clean markdown via a reader service (Jina r.jina.ai)' },
+    { tier: 3, name: 'tavily', solves: 'origin blocks a plain fetch — provider-side extract/search (needs TAVILY_API_KEY)' },
+    { tier: 4, name: 'archive', solves: 'origin is down or removed — public archived snapshot (Wayback Machine)' },
+];
+const DEFAULTS = {
+    fetchImpl: ((url, init) => fetch(url, init)),
+    userAgent: SANOOK_USER_AGENT,
+    timeoutMs: 15_000,
+    maxBytes: 600_000,
+    respectRobots: true,
+    allowReader: true,
+    readerBase: 'https://r.jina.ai/',
+    allowArchive: true,
+    allowPrivateHosts: false,
+    tavilyBase: 'https://api.tavily.com',
+    maxLinks: 25,
+    maxHeadings: 20,
+};
+function resolveOptions(options) {
+    return { ...DEFAULTS, ...options, fetchImpl: options.fetchImpl ?? DEFAULTS.fetchImpl };
+}
+// ── SSRF guard ──────────────────────────────────────────────────────────────
+// Block loopback / private / link-local / metadata hosts by default so an agent
+// fetching an arbitrary URL can't reach internal services.
+export function isPrivateHost(hostname) {
+    // strip brackets, lowercase, drop a single trailing dot (FQDN form evades endsWith otherwise)
+    const host = hostname
+        .toLowerCase()
+        .replace(/^\[|\]$/g, '')
+        .replace(/\.$/, '');
+    // IPv4-mapped IPv6 (::ffff:127.0.0.1 or ::ffff:7f00:1) — unwrap and re-check the embedded v4
+    const mapped = host.match(/^::ffff:(.+)$/i);
+    if (mapped) {
+        const inner = mapped[1];
+        if (/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(inner))
+            return isPrivateHost(inner);
+        const hex = inner.match(/^([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i);
+        if (hex) {
+            const hi = parseInt(hex[1], 16);
+            const lo = parseInt(hex[2], 16);
+            return isPrivateHost(`${(hi >> 8) & 255}.${hi & 255}.${(lo >> 8) & 255}.${lo & 255}`);
+        }
+        return true; // unrecognised mapped form → fail closed
+    }
+    if (host === 'localhost' || host.endsWith('.localhost') || host.endsWith('.local') || host.endsWith('.internal'))
+        return true;
+    if (host === '0.0.0.0' || host === '::' || host === '::1')
+        return true;
+    // IPv4
+    const v4 = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
+    if (v4) {
+        const [a, b] = [Number(v4[1]), Number(v4[2])];
+        if (a === 127 || a === 10 || a === 0)
+            return true;
+        if (a === 192 && b === 168)
+            return true;
+        if (a === 169 && b === 254)
+            return true; // link-local + cloud metadata 169.254.169.254
+        if (a === 172 && b >= 16 && b <= 31)
+            return true;
+        if (a === 100 && b >= 64 && b <= 127)
+            return true; // CGNAT
+    }
+    // IPv6 unique-local / link-local
+    if (/^f[cd][0-9a-f]{2}:/i.test(host) || /^fe80:/i.test(host))
+        return true;
+    return false;
+}
+// ── robots.txt ──────────────────────────────────────────────────────────────
+// Minimal but correct-enough parser: honour the most specific matching group for
+// our UA token, falling back to `*`. Returns whether `path` is allowed to fetch.
+export function isAllowedByRobots(robotsTxt, path, uaToken = BRAND.cliName) {
+    const groups = [];
+    let current = null;
+    let lastWasAgent = false;
+    for (const rawLine of robotsTxt.split(/\r?\n/)) {
+        const line = rawLine.replace(/#.*$/, '').trim();
+        if (!line)
+            continue;
+        const idx = line.indexOf(':');
+        if (idx < 0)
+            continue;
+        const field = line.slice(0, idx).trim().toLowerCase();
+        const value = line.slice(idx + 1).trim();
+        if (field === 'user-agent') {
+            if (!current || !lastWasAgent) {
+                current = { agents: [], rules: [] };
+                groups.push(current);
+            }
+            current.agents.push(value.toLowerCase());
+            lastWasAgent = true;
+        }
+        else if ((field === 'allow' || field === 'disallow') && current) {
+            current.rules.push({ allow: field === 'allow', path: value });
+            lastWasAgent = false;
+        }
+        else {
+            lastWasAgent = false;
+        }
+    }
+    // Match the single product token (e.g. "sanook") against agent lines, not the full
+    // descriptive UA — otherwise a group named "web"/"cli"/"fetch" would falsely apply.
+    const ua = uaToken.toLowerCase().split(/[\s/]/)[0];
+    const specific = groups.find((g) => g.agents.some((a) => a !== '*' && a !== '' && (ua === a || ua.startsWith(a))));
+    const wildcard = groups.find((g) => g.agents.includes('*'));
+    const group = specific ?? wildcard;
+    if (!group)
+        return true;
+    // Longest matching pattern wins; an empty Disallow means "allow all".
+    let decision = true;
+    let best = -1;
+    for (const rule of group.rules) {
+        if (!rule.path)
+            continue; // empty path (e.g. "Disallow:") = no-op / allow all
+        if (rule.path.length > best && robotsRuleToRegex(rule.path).test(path)) {
+            best = rule.path.length;
+            decision = rule.allow;
+        }
+    }
+    return decision;
+}
+// Convert a robots.txt path rule (with * wildcards and optional $ end-anchor) to a regex.
+function robotsRuleToRegex(rule) {
+    const anchored = rule.endsWith('$');
+    const body = (anchored ? rule.slice(0, -1) : rule)
+        .split('*')
+        .map((segment) => segment.replace(/[.+?^${}()|[\]\\]/g, '\\$&'))
+        .join('.*');
+    return new RegExp(`^${body}${anchored ? '$' : ''}`);
+}
+// ── HTML helpers ─────────────────────────────────────────────────────────────
+function decodeEntities(input) {
+    return input
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#0*39;|&apos;/g, "'")
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&#(\d+);/g, (_, n) => safeCodePoint(Number(n)))
+        .replace(/&#x([0-9a-f]+);/gi, (_, h) => safeCodePoint(parseInt(h, 16)));
+}
+function safeCodePoint(n) {
+    try {
+        return Number.isFinite(n) && n > 0 && n <= 0x10ffff ? String.fromCodePoint(n) : '';
+    }
+    catch {
+        return '';
+    }
+}
+function stripTags(html) {
+    return decodeEntities(html.replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim();
+}
+function withoutNoise(html) {
+    return html
+        .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ')
+        .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ')
+        .replace(/<!--[\s\S]*?-->/g, ' ')
+        .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, ' ');
+}
+function getAttr(tag, name) {
+    const m = tag.match(new RegExp(`\\b${name}\\s*=\\s*("([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i'));
+    return m ? decodeEntities(m[2] ?? m[3] ?? m[4] ?? '') : undefined;
+}
+function collectMeta(html) {
+    const out = new Map();
+    for (const m of html.matchAll(/<meta\b[^>]*>/gi)) {
+        const tag = m[0];
+        const key = (getAttr(tag, 'name') ?? getAttr(tag, 'property') ?? getAttr(tag, 'itemprop'))?.toLowerCase();
+        const content = getAttr(tag, 'content');
+        if (key && content && !out.has(key))
+            out.set(key, content);
+    }
+    return out;
+}
+function extractLinks(html, baseUrl, max) {
+    const seen = new Set();
+    const out = [];
+    for (const m of html.matchAll(/<a\b[^>]*\bhref\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))[^>]*>([\s\S]*?)<\/a>/gi)) {
+        const rawHref = decodeEntities(m[2] ?? m[3] ?? m[4] ?? '').trim();
+        if (!rawHref || rawHref.startsWith('#') || /^(javascript|mailto|tel):/i.test(rawHref))
+            continue;
+        let href;
+        try {
+            href = new URL(rawHref, baseUrl).href;
+        }
+        catch {
+            continue;
+        }
+        if (!/^https?:/i.test(href) || seen.has(href))
+            continue;
+        seen.add(href);
+        out.push({ text: stripTags(m[5]).slice(0, 120) || href, href });
+        if (out.length >= max)
+            break;
+    }
+    return out;
+}
+function extractJsonLdTypes(html) {
+    const types = new Set();
+    const visit = (node) => {
+        if (Array.isArray(node)) {
+            node.forEach(visit);
+            return;
+        }
+        if (!node || typeof node !== 'object')
+            return;
+        const rec = node;
+        const t = rec['@type'];
+        if (typeof t === 'string')
+            types.add(t);
+        else if (Array.isArray(t))
+            t.forEach((x) => typeof x === 'string' && types.add(x));
+        if (Array.isArray(rec['@graph']))
+            rec['@graph'].forEach(visit);
+    };
+    for (const m of html.matchAll(/<script\b[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
+        try {
+            visit(JSON.parse(m[1].trim()));
+        }
+        catch {
+            /* malformed JSON-LD — skip */
+        }
+    }
+    return [...types].slice(0, 12);
+}
+export function extractStructure(html, baseUrl, options = {}) {
+    const opts = resolveOptions(options);
+    const meta = collectMeta(html);
+    const metaOf = (...keys) => {
+        for (const k of keys) {
+            const v = meta.get(k);
+            if (v)
+                return v;
+        }
+        return undefined;
+    };
+    const titleTag = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    const title = (titleTag ? stripTags(titleTag[1]) : undefined) || metaOf('og:title', 'twitter:title');
+    const description = metaOf('description', 'og:description', 'twitter:description');
+    const siteName = metaOf('og:site_name', 'application-name');
+    const lang = html.match(/<html\b[^>]*\blang\s*=\s*["']([^"']+)["']/i)?.[1];
+    const headings = [];
+    for (const m of withoutNoise(html).matchAll(/<h[1-3]\b[^>]*>([\s\S]*?)<\/h[1-3]>/gi)) {
+        const text = stripTags(m[1]);
+        if (text && !headings.includes(text))
+            headings.push(text);
+        if (headings.length >= opts.maxHeadings)
+            break;
+    }
+    const wordCount = withoutNoise(html)
+        .replace(/<[^>]+>/g, ' ')
+        .split(/\s+/)
+        .filter((w) => w.length > 1).length;
+    let host = baseUrl;
+    try {
+        host = new URL(baseUrl).host;
+    }
+    catch {
+        /* keep raw */
+    }
+    return {
+        title,
+        description,
+        siteName,
+        lang,
+        headings,
+        links: extractLinks(html, baseUrl, opts.maxLinks),
+        jsonLdTypes: extractJsonLdTypes(html),
+        wordCount,
+        summary: buildSummary({ title, description, siteName, headings, host }),
+    };
+}
+function buildSummary(parts) {
+    const lead = parts.title || parts.siteName || parts.host;
+    const tail = parts.description || parts.headings[0] || '';
+    const text = tail ? `${lead} — ${tail}` : lead;
+    return text.length > 280 ? `${text.slice(0, 277)}…` : text;
+}
+// Markers of a bot-challenge / JS-required interstitial. If we see one we treat the
+// tier as a FAILURE and fall through — this is RECOGNISING a block, never bypassing it.
+// Unambiguous interstitial markers — if present, it's a challenge page regardless of length.
+const STRONG_CHALLENGE_MARKERS = [
+    'cf-browser-verification',
+    'challenge-platform',
+    '/cdn-cgi/challenge',
+    'attention required! | cloudflare',
+    'checking your browser before accessing',
+    'enable javascript and cookies to continue',
+    'ddos protection by',
+];
+// Phrases that also appear in legit prose — only count them when the page has little real content.
+const WEAK_CHALLENGE_MARKERS = ['just a moment', 'verifying you are human', 'please verify you are a human', 'requiring captcha'];
+export function looksBlocked(text) {
+    const lc = text.toLowerCase(); // scan the whole (already size-capped) body, not just the head
+    if (STRONG_CHALLENGE_MARKERS.some((marker) => lc.includes(marker)))
+        return true;
+    if (WEAK_CHALLENGE_MARKERS.some((marker) => lc.includes(marker))) {
+        const words = text.replace(/<[^>]+>/g, ' ').split(/\s+/).filter((w) => w.length > 1).length;
+        return words < 120; // a real article that merely mentions the phrase has far more content
+    }
+    return false;
+}
+function looksHtml(contentType, body) {
+    if (contentType && /(text\/html|application\/xhtml)/i.test(contentType))
+        return true;
+    if (contentType && /(json|pdf|image\/|application\/octet)/i.test(contentType))
+        return false;
+    return /<html[\s>]|<!doctype html|<head[\s>]|<body[\s>]/i.test(body.slice(0, 4000));
+}
+const MAX_REDIRECTS = 5;
+function urlGuardError(parsed, opts) {
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:')
+        return `unsupported protocol: ${parsed.protocol}`;
+    if (!opts.allowPrivateHosts && isPrivateHost(parsed.hostname))
+        return `blocked internal/loopback host: ${parsed.hostname} (SSRF guard)`;
+    return null;
+}
+// ── single GET with timeout, size cap, and SSRF-safe MANUAL redirect handling ─
+// Every redirect hop is re-validated against the SSRF guard, so a public URL that
+// 30x-redirects to a private/loopback/metadata host is rejected, not followed.
+async function doGet(url, opts, extraHeaders = {}) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), opts.timeoutMs);
+    try {
+        let currentUrl = url;
+        for (let hop = 0;; hop++) {
+            let parsed;
+            try {
+                parsed = new URL(currentUrl);
+            }
+            catch {
+                return { error: 'invalid redirect URL' };
+            }
+            const guard = urlGuardError(parsed, opts);
+            if (guard)
+                return { error: guard };
+            const res = await opts.fetchImpl(currentUrl, {
+                redirect: 'manual',
+                signal: controller.signal,
+                headers: { 'user-agent': opts.userAgent, accept: 'text/html,application/xhtml+xml,text/plain,*/*', ...extraHeaders },
+            });
+            if (res.status >= 300 && res.status < 400) {
+                const location = res.headers.get('location');
+                if (!location)
+                    return { error: `${res.status} redirect without Location`, status: res.status };
+                if (hop >= MAX_REDIRECTS)
+                    return { error: `too many redirects (>${MAX_REDIRECTS})`, status: res.status };
+                currentUrl = new URL(location, currentUrl).href; // re-validated at top of next hop
+                continue;
+            }
+            if (!res.ok) {
+                return { error: `${res.status}`, status: res.status, retryAfter: res.headers.get('retry-after') ?? undefined };
+            }
+            const declared = Number(res.headers.get('content-length') ?? '');
+            if (Number.isFinite(declared) && declared > opts.maxBytes) {
+                return { error: `response too large (${declared} bytes > ${opts.maxBytes} cap)`, status: res.status };
+            }
+            const body = (await res.text()).slice(0, opts.maxBytes);
+            return { res, body, finalUrl: currentUrl };
+        }
+    }
+    catch (e) {
+        return { error: e.name === 'AbortError' ? `timeout >${opts.timeoutMs}ms` : e.message };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+async function checkRobots(parsed, opts, attempts) {
+    const robotsUrl = `${parsed.protocol}//${parsed.host}/robots.txt`;
+    const got = await doGet(robotsUrl, { ...opts, respectRobots: false });
+    if ('error' in got) {
+        // No robots.txt (or unreachable) → not disallowed.
+        attempts.push({ tier: 'robots', ok: true, detail: `no robots.txt readable (${got.error}) — treated as allowed` });
+        return true;
+    }
+    const allowed = isAllowedByRobots(got.body, parsed.pathname + parsed.search, BRAND.cliName);
+    attempts.push({ tier: 'robots', ok: allowed, detail: allowed ? 'allowed by robots.txt' : `disallowed by robots.txt for ${parsed.pathname}` });
+    return allowed;
+}
+async function tierDirect(parsed, opts, attempts) {
+    const got = await doGet(parsed.href, opts);
+    if ('error' in got) {
+        const extra = got.retryAfter ? ` (retry-after: ${got.retryAfter})` : '';
+        attempts.push({ tier: 'direct', ok: false, detail: `direct fetch failed: ${got.error}${extra}`, status: got.status });
+        return null;
+    }
+    const contentType = got.res.headers.get('content-type') ?? undefined;
+    if (looksBlocked(got.body)) {
+        attempts.push({ tier: 'direct', ok: false, detail: 'got a bot-challenge/JS interstitial — not bypassing, falling through', status: got.res.status });
+        return null;
+    }
+    if (looksHtml(contentType, got.body)) {
+        const structure = extractStructure(got.body, got.finalUrl, opts);
+        attempts.push({ tier: 'direct', ok: true, detail: `direct HTML — ${structure.wordCount} words, ${structure.headings.length} headings`, status: got.res.status });
+        return { winningTier: 'direct', finalUrl: got.finalUrl, status: got.res.status, contentType, structure };
+    }
+    attempts.push({ tier: 'direct', ok: true, detail: `direct non-HTML (${contentType ?? 'unknown'}) — returned as text`, status: got.res.status });
+    return { winningTier: 'direct', finalUrl: got.finalUrl, status: got.res.status, contentType, content: stripTags(got.body).slice(0, opts.maxBytes) };
+}
+async function tierReader(parsed, opts, attempts) {
+    // Jina reader takes the full target URL appended to its base; do NOT pre-encode the whole URL
+    // (encodeURI/encodeURIComponent would break r.jina.ai's path parsing of query strings).
+    const readerUrl = `${opts.readerBase.replace(/\/$/, '')}/${parsed.href}`;
+    const got = await doGet(readerUrl, opts, { accept: 'text/plain', 'x-return-format': 'markdown' });
+    if ('error' in got) {
+        attempts.push({ tier: 'reader', ok: false, detail: `reader (${opts.readerBase}) failed: ${got.error}`, status: got.status });
+        return null;
+    }
+    const content = got.body.trim();
+    if (!content) {
+        attempts.push({ tier: 'reader', ok: false, detail: 'reader returned empty body' });
+        return null;
+    }
+    if (looksBlocked(content)) {
+        attempts.push({ tier: 'reader', ok: false, detail: 'reader returned a bot-challenge interstitial — not bypassing, falling through', status: got.res.status });
+        return null;
+    }
+    attempts.push({ tier: 'reader', ok: true, detail: `reader markdown — ${content.length} chars`, status: got.res.status });
+    return { winningTier: 'reader', finalUrl: parsed.href, status: got.res.status, contentType: 'text/markdown', content, structure: structureFromMarkdown(content, parsed.host) };
+}
+function structureFromMarkdown(markdown, host) {
+    const titleLine = markdown.match(/^Title:\s*(.+)$/im)?.[1] ?? markdown.match(/^#\s+(.+)$/m)?.[1];
+    const headings = [...markdown.matchAll(/^#{1,3}\s+(.+)$/gm)].map((m) => m[1].trim()).slice(0, 20);
+    const firstPara = markdown.split(/\n\s*\n/).map((p) => p.trim()).find((p) => p && !p.startsWith('#') && !/^(title|url source):/i.test(p));
+    const title = titleLine?.trim();
+    return {
+        title,
+        description: firstPara?.slice(0, 280),
+        headings,
+        links: [],
+        jsonLdTypes: [],
+        wordCount: markdown.split(/\s+/).filter((w) => w.length > 1).length,
+        summary: buildSummary({ title, description: firstPara, headings, host }),
+    };
+}
+async function tierTavily(parsed, opts, attempts) {
+    if (!opts.tavilyApiKey)
+        return null;
+    try {
+        const res = await opts.fetchImpl(`${opts.tavilyBase.replace(/\/$/, '')}/extract`, {
+            method: 'POST',
+            signal: AbortSignal.timeout(opts.timeoutMs),
+            headers: { 'content-type': 'application/json', authorization: `Bearer ${opts.tavilyApiKey}` },
+            body: JSON.stringify({ urls: [parsed.href] }),
+        });
+        if (!res.ok) {
+            attempts.push({ tier: 'tavily', ok: false, detail: `tavily extract failed: ${res.status}`, status: res.status });
+            return null;
+        }
+        const json = JSON.parse(await res.text());
+        const content = json.results?.[0]?.raw_content || json.results?.[0]?.content;
+        if (!content) {
+            attempts.push({ tier: 'tavily', ok: false, detail: 'tavily extract returned no content' });
+            return null;
+        }
+        if (looksBlocked(content)) {
+            attempts.push({ tier: 'tavily', ok: false, detail: 'tavily returned a bot-challenge interstitial — not bypassing, falling through', status: res.status });
+            return null;
+        }
+        attempts.push({ tier: 'tavily', ok: true, detail: `tavily extract — ${content.length} chars`, status: res.status });
+        return { winningTier: 'tavily', finalUrl: parsed.href, status: res.status, contentType: 'text/plain', content: content.slice(0, opts.maxBytes), structure: structureFromMarkdown(content, parsed.host) };
+    }
+    catch (e) {
+        attempts.push({ tier: 'tavily', ok: false, detail: `tavily extract error: ${e.message}` });
+        return null;
+    }
+}
+async function tierArchive(parsed, opts, attempts) {
+    const availUrl = `https://archive.org/wayback/available?url=${encodeURIComponent(parsed.href)}`;
+    const avail = await doGet(availUrl, { ...opts, respectRobots: false }, { accept: 'application/json' });
+    if ('error' in avail) {
+        attempts.push({ tier: 'archive', ok: false, detail: `wayback lookup failed: ${avail.error}` });
+        return null;
+    }
+    let snapshotUrl;
+    let timestamp;
+    try {
+        const json = JSON.parse(avail.body);
+        const closest = json.archived_snapshots?.closest;
+        if (closest?.available && closest.url) {
+            snapshotUrl = closest.url;
+            timestamp = closest.timestamp;
+        }
+    }
+    catch {
+        /* fall through */
+    }
+    if (!snapshotUrl) {
+        attempts.push({ tier: 'archive', ok: false, detail: 'no Wayback snapshot available' });
+        return null;
+    }
+    const snap = await doGet(snapshotUrl, { ...opts, respectRobots: false });
+    if ('error' in snap) {
+        attempts.push({ tier: 'archive', ok: false, detail: `snapshot fetch failed: ${snap.error}`, status: snap.status });
+        return null;
+    }
+    if (looksBlocked(snap.body)) {
+        attempts.push({ tier: 'archive', ok: false, detail: 'archived snapshot is itself a challenge page', status: snap.res.status });
+        return null;
+    }
+    const structure = extractStructure(snap.body, snapshotUrl, opts);
+    attempts.push({ tier: 'archive', ok: true, detail: `Wayback snapshot ${timestamp ?? ''} — ${structure.wordCount} words`, status: snap.res.status });
+    return { winningTier: 'archive', finalUrl: snapshotUrl, status: snap.res.status, contentType: snap.res.headers.get('content-type') ?? undefined, structure };
+}
+function blockedNote(opts) {
+    const tried = ['direct fetch', opts.allowReader ? 'reader service' : '', opts.tavilyApiKey ? 'Tavily extract' : '', opts.allowArchive ? 'Wayback archive' : '']
+        .filter(Boolean)
+        .join(', ');
+    return `ดึงไม่ได้ด้วยวิธีที่ถูกกติกา (${tried}). หน้านี้อาจต้อง JS render, login, หรือมี anti-bot. Sanook จะไม่ทำ evasion (${REFUSED_TECHNIQUES.join(', ')}) — ลองหา official API / RSS / sitemap หรือขออนุญาตเจ้าของเว็บแทน`;
+}
+/**
+ * Fetch a public web page through an ordered, ethical fallback ladder and return
+ * a rough structural understanding of it. Never attempts any REFUSED_TECHNIQUES.
+ */
+export async function fetchWeb(rawUrl, options = {}) {
+    const opts = resolveOptions(options);
+    const attempts = [];
+    const result = { url: rawUrl, ok: false, attempts, refused: REFUSED_TECHNIQUES };
+    let parsed;
+    try {
+        parsed = new URL(rawUrl);
+    }
+    catch {
+        attempts.push({ tier: 'preflight', ok: false, detail: 'invalid URL' });
+        result.note = 'URL ไม่ถูกต้อง';
+        return result;
+    }
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+        attempts.push({ tier: 'preflight', ok: false, detail: `unsupported protocol: ${parsed.protocol}` });
+        result.note = 'รองรับเฉพาะ http/https';
+        return result;
+    }
+    if (!opts.allowPrivateHosts && isPrivateHost(parsed.hostname)) {
+        attempts.push({ tier: 'preflight', ok: false, detail: `blocked internal/loopback host: ${parsed.hostname} (SSRF guard)` });
+        result.note = 'host ภายใน/loopback ถูกบล็อกกัน SSRF — ใช้ allowPrivateHosts ถ้าตั้งใจจริง';
+        return result;
+    }
+    if (opts.respectRobots) {
+        const allowed = await checkRobots(parsed, opts, attempts);
+        if (!allowed) {
+            result.note = `robots.txt ของ ${parsed.host} ห้าม fetch path นี้ — เคารพ robots, ไม่ดึงต่อ. ใช้ official API/ขออนุญาตแทน`;
+            return result;
+        }
+    }
+    const ladder = [tierDirect];
+    if (opts.allowReader)
+        ladder.push(tierReader);
+    if (opts.tavilyApiKey)
+        ladder.push(tierTavily);
+    if (opts.allowArchive)
+        ladder.push(tierArchive);
+    for (const tier of ladder) {
+        const outcome = await tier(parsed, opts, attempts);
+        if (outcome)
+            return { ...result, ok: true, ...outcome };
+    }
+    result.note = blockedNote(opts);
+    return result;
+}
+export async function tavilySearch(query, options) {
+    const fetchImpl = options.fetchImpl ?? DEFAULTS.fetchImpl;
+    const base = (options.tavilyBase ?? DEFAULTS.tavilyBase).replace(/\/$/, '');
+    const res = await fetchImpl(`${base}/search`, {
+        method: 'POST',
+        signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULTS.timeoutMs),
+        headers: { 'content-type': 'application/json', authorization: `Bearer ${options.apiKey}` },
+        body: JSON.stringify({ query, max_results: options.maxResults ?? 5 }),
+    });
+    if (!res.ok)
+        throw new Error(`tavily search ${res.status}`);
+    const json = JSON.parse(await res.text());
+    return (json.results ?? []).slice(0, options.maxResults ?? 5);
+}
+// ── rendering ────────────────────────────────────────────────────────────────
+export function renderWebFetchResult(result) {
+    const lines = [`web fetch: ${result.url}`];
+    if (result.finalUrl && result.finalUrl !== result.url)
+        lines.push(`final url: ${result.finalUrl}`);
+    lines.push(`result: ${result.ok ? `OK via tier "${result.winningTier}"` : 'BLOCKED (no ethical tier succeeded)'}`);
+    if (result.status)
+        lines.push(`status: ${result.status}${result.contentType ? ` · ${result.contentType}` : ''}`);
+    if (result.structure) {
+        const s = result.structure;
+        lines.push('', 'structure:');
+        if (s.title)
+            lines.push(`  title: ${s.title}`);
+        if (s.siteName)
+            lines.push(`  site: ${s.siteName}`);
+        if (s.description)
+            lines.push(`  description: ${s.description}`);
+        if (s.lang)
+            lines.push(`  lang: ${s.lang}`);
+        lines.push(`  words: ${s.wordCount}`);
+        if (s.jsonLdTypes.length)
+            lines.push(`  schema.org types: ${s.jsonLdTypes.join(', ')}`);
+        if (s.headings.length) {
+            lines.push(`  headings (${s.headings.length}):`);
+            for (const h of s.headings.slice(0, 10))
+                lines.push(`    - ${h}`);
+        }
+        if (s.links.length) {
+            lines.push(`  links (${s.links.length}):`);
+            for (const l of s.links.slice(0, 10))
+                lines.push(`    - ${l.text} → ${l.href}`);
+        }
+        lines.push('', `summary: ${s.summary}`);
+    }
+    else if (result.content) {
+        lines.push('', `content (${result.content.length} chars), first 600:`, result.content.slice(0, 600));
+    }
+    lines.push('', 'attempts:');
+    for (const a of result.attempts)
+        lines.push(`  ${a.ok ? '✓' : '✗'} ${a.tier}: ${a.detail}`);
+    if (result.note)
+        lines.push('', `note: ${result.note}`);
+    lines.push('', `never attempted (policy): ${result.refused.join(', ')}`);
+    return lines.join('\n');
+}