smart-web-mcp 0.44.1 → 1.0.0

package/dist/smartfetch.js

@@ -1,5 +1,5 @@
 import { absolutizeUrl, assetFromUrl, asString, cleanLinks, dedupeUrls, configFlag, extractAnchorHrefs, extractUrls, inferTarget, needsBotChallengeBrowserUpgrade, needsDynamicCrawl, needsSpaBrowserFallback, resolveValidatedUrl, validateOutboundUrl, } from "./shared.js";
-import { createAssessment } from "./assessment.js";
+import { createAssessment, hasTerminalSafetyBlock } from "./assessment.js";
 import { runBrowserSession, resolveProfileDir } from "./browser-session.js";
 import { projectSmartfetchOutput, resolveSmartfetchBudgetPolicy } from "./extraction/budget.js";
 import { configText, configValue } from "./settings.js";
@@ -14,6 +14,10 @@ import { maybeUseJinaReader } from "./smartfetch/jina-reader.js";
 import { runCoupang } from "./korea/coupang.js";
 import { assessSmartfetchOutput, buildAcquireStage, buildNormalizeStage, buildResolveStage } from "./smartfetch/pipeline.js";
 import { resolveSmartfetchProvider } from "./smartfetch/providers/index.js";
+import { naverBlogMobilePostUrl } from "./smartfetch/providers/naver-blog.js";
+import { NAVER_CAFE_READY_SELECTOR, naverCafeMobilePostUrl } from "./smartfetch/providers/naver-cafe-native.js";
+import { exportComments } from "./comment-export.js";
+import { exportBody } from "./body-export.js";
 const FAILURE_CACHE_TTL_MS = 30_000;
 const IMPIT_MAX_REDIRECTS = 5;
 const TEXT_RENDER_LIMITS = {
@@ -26,14 +30,42 @@ const TEXT_RENDER_LIMITS = {
     handoffLinks: 8,
     assets: 10,
 };
-/** Inflight coalescing: deduplicate concurrent smartfetch calls to the same URL. */
-const inflightSmartfetch = new Map();
 function isRedirect(status) {
     return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
 }
 function allowPrivateHosts() {
     return configFlag("network.allowPrivateHosts");
 }
+function nativeNaverTarget(url) {
+    if (naverBlogMobilePostUrl(url))
+        return "naver_blog";
+    if (naverCafeMobilePostUrl(url))
+        return "naver_cafe_post";
+    return undefined;
+}
+function operatorRecoveryReason(result) {
+    if (!configText("fetch.browserCdpUrl") || result.ok || hasTerminalSafetyBlock([result.error]))
+        return "";
+    // An explicit cooldown is not an invitation to retry in another client.
+    if (result.http_status === 429 || result.retry_after)
+        return "";
+    if (result.http_status === 403 || (result.http_status && result.http_status >= 500)) {
+        return `direct HTTP ${result.http_status} — continued in approved operator browser`;
+    }
+    if (["redirect_resolution_failed", "plain_fetch_exception", "impit_fetch_exception", "direct_acquisition_timeout"].includes(result.error?.code || "")) {
+        return "direct transport failed — continued in approved operator browser";
+    }
+    return "";
+}
+function preflightCooldown(result, method) {
+    if (result.http_status !== 429 && !result.retry_after)
+        return undefined;
+    return {
+        ok: false, method, content: "", links: [], final_url: result.url,
+        http_status: result.http_status, retry_after: result.retry_after,
+        error: { category: "block", code: "direct_retry_after", message: `status=${result.http_status}${result.retry_after ? `; Retry-After=${result.retry_after}` : ""}` },
+    };
+}
 function compactText(value, maxChars) {
     const text = String(value || "").trim();
     if (text.length <= maxChars)
@@ -108,11 +140,36 @@ function redirectUrl(url, headers, body) {
     }
     return "";
 }
-async function runNativeFetch(url, timeoutMs) {
+class FetchDeadlineError extends Error {
+    constructor() { super("The original smartfetch deadline expired"); }
+}
+function remainingFetchTime(deadlineAt) {
+    const remaining = deadlineAt - Date.now();
+    if (remaining <= 0)
+        throw new FetchDeadlineError();
+    return remaining;
+}
+async function withinFetchDeadline(deadlineAt, work) {
+    const controller = new AbortController();
+    const remaining = remainingFetchTime(deadlineAt);
+    let timer;
+    const expiry = new Promise((_resolve, reject) => {
+        timer = setTimeout(() => { controller.abort(); reject(new FetchDeadlineError()); }, remaining);
+    });
+    try {
+        return await Promise.race([work(controller.signal), expiry]);
+    }
+    finally {
+        clearTimeout(timer);
+        controller.abort();
+    }
+}
+async function runNativeFetch(url, deadlineAt, signal) {
     const headers = directFetchHeaders(url);
     const resolved = await resolveValidatedUrl(url, {
         allowPrivateHosts: allowPrivateHosts(),
-        timeoutMs,
+        timeoutMs: remainingFetchTime(deadlineAt),
+        signal,
         headers,
     });
     if (!resolved.ok) {
@@ -128,18 +185,26 @@ async function runNativeFetch(url, timeoutMs) {
             },
         };
     }
+    const cooldown = preflightCooldown(resolved, "plain_fetch");
+    if (cooldown)
+        return cooldown;
     const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    const timer = setTimeout(() => controller.abort(), remainingFetchTime(deadlineAt));
     try {
-        const response = await fetch(resolved.url, { signal: controller.signal, headers, redirect: "error" });
-        const body = await response.text();
+        const requestSignal = AbortSignal.any([signal, controller.signal]);
+        requestSignal.throwIfAborted();
+        const response = await fetch(resolved.url, { signal: requestSignal, headers, redirect: "error" });
         if (!response.ok) {
+            // Cleanup must not delay publishing an observed terminal status.
+            void response.body?.cancel().catch(() => { });
             return {
                 ok: false,
                 method: "plain_fetch",
                 content: "",
                 links: [],
                 final_url: resolved.url,
+                http_status: response.status,
+                retry_after: header(response.headers, "retry-after"),
                 error: {
                     category: response.status === 403 || response.status === 429 ? "block" : "unavailable",
                     code: "plain_fetch_failed",
@@ -147,6 +212,8 @@ async function runNativeFetch(url, timeoutMs) {
                 },
             };
         }
+        const body = await response.text();
+        remainingFetchTime(deadlineAt);
         const links = [...extractAnchorHrefs(body), ...extractUrls(body)].map((item) => absolutizeUrl(item, resolved.url) || item).filter(Boolean);
         return { ok: true, method: "plain_fetch", content: body, links: dedupeUrls(links), final_url: resolved.url };
     }
@@ -169,14 +236,17 @@ async function runNativeFetch(url, timeoutMs) {
         clearTimeout(timer);
     }
 }
-async function runImpitFetch(url, timeoutMs) {
+async function runImpitFetch(url, deadlineAt, signal) {
     const Impit = await getImpit();
+    signal.throwIfAborted();
+    remainingFetchTime(deadlineAt);
     if (!Impit)
-        return runNativeFetch(url, timeoutMs);
+        return runNativeFetch(url, deadlineAt, signal);
     const headers = impitFetchHeaders(url);
     const resolved = await resolveValidatedUrl(url, {
         allowPrivateHosts: allowPrivateHosts(),
-        timeoutMs,
+        timeoutMs: remainingFetchTime(deadlineAt),
+        signal,
         headers,
     });
     if (!resolved.ok) {
@@ -192,12 +262,19 @@ async function runImpitFetch(url, timeoutMs) {
             },
         };
     }
+    const cooldown = preflightCooldown(resolved, "impit_fetch");
+    if (cooldown)
+        return cooldown;
     let currentUrl = resolved.url;
     try {
-        const client = new Impit({ browser: "chrome", timeout: timeoutMs });
+        const client = new Impit({ browser: "chrome", timeout: remainingFetchTime(deadlineAt) });
         for (let hop = 0; hop <= IMPIT_MAX_REDIRECTS; hop++) {
-            const response = await client.fetch(currentUrl, { headers, redirect: "error" });
-            const body = await response.text();
+            signal.throwIfAborted();
+            const response = await client.fetch(currentUrl, { headers, redirect: "error", signal, timeout: remainingFetchTime(deadlineAt) });
+            // Preserve status/cooldown even if an error response body is interrupted.
+            const body = response.ok || isRedirect(response.status) ? await response.text() : "";
+            if (response.ok || isRedirect(response.status))
+                remainingFetchTime(deadlineAt);
             if (response.ok) {
                 const finalUrl = asString(response.url).trim() || currentUrl;
                 const base = finalUrl || currentUrl;
@@ -211,12 +288,15 @@ async function runImpitFetch(url, timeoutMs) {
                 };
             }
             if (!isRedirect(response.status)) {
+                void response.body?.cancel().catch(() => { });
                 return {
                     ok: false,
                     method: "impit_fetch",
                     content: "",
                     links: [],
                     final_url: asString(response.url).trim() || currentUrl,
+                    http_status: response.status,
+                    retry_after: header(response.headers, "retry-after"),
                     error: {
                         category: response.status === 403 || response.status === 429 ? "block" : "unavailable",
                         code: "impit_fetch_failed",
@@ -316,6 +396,7 @@ async function runPlaywrightFetch(url, timeoutMs, target, profileDir, waitForSel
     const result = await runBrowserSession({
         url,
         timeoutMs,
+        ...(!profileDir && configText("fetch.browserCdpUrl") ? { cdpUrl: configText("fetch.browserCdpUrl") } : {}),
         allowPrivateHosts: allowPrivateHosts(),
         waitUntil: choosePlaywrightWaitUntil(target, url),
         extraHeaders,
@@ -462,6 +543,8 @@ const HEADROOM_BALANCED_MIN = 60_000;
 const PREFERRED_BALANCED_MIN = 30_000;
 const PREFERRED_FULL_MIN = 120_000;
 function asNonNegativeInteger(value) {
+    if (value === null || value === undefined)
+        return null;
     const parsed = Number(value);
     if (!Number.isFinite(parsed) || parsed < 0)
         return null;
@@ -584,28 +667,32 @@ function isDocumentLikeAsset(asset) {
     return asset?.kind === "document" || asset?.kind === "archive" || asset?.kind === "attachment";
 }
 export async function runSmartfetch(options, runtime = {}) {
-    // Inflight coalescing: deduplicate calls only when the URL and contextual
-    // commerce queries are identical. Distinct product hints must not share a result.
-    const coalesceKey = !options.download && !options.profile
-        ? `${options.url}\u0000${JSON.stringify(options.referenceQueries || [])}`
-        : "";
-    if (coalesceKey) {
-        const inflight = inflightSmartfetch.get(coalesceKey);
-        if (inflight)
-            return inflight;
+    const timeoutMs = Math.max(3000, Math.floor(Number(options.timeoutMs || 25000)));
+    const deadlineAt = Date.now() + timeoutMs;
+    const output = baseOutput(options.url, inferTarget(options.url, options.target || "auto"));
+    try {
+        return await runSmartfetchTask(options, runtime, deadlineAt, output);
     }
-    const promise = runSmartfetchInner(options, runtime);
-    if (coalesceKey) {
-        inflightSmartfetch.set(coalesceKey, promise);
-        promise.finally(() => { inflightSmartfetch.delete(coalesceKey); });
+    catch (error) {
+        if (!(error instanceof FetchDeadlineError))
+            throw error;
+        output.partial = true;
+        output.errors.push({ category: "timeout", code: "smartfetch_deadline_exceeded", message: error.message });
+        output.assessment = createAssessment({ confidence: "low", blocked_reason: error.message, signals: ["retrieval:deadline_exceeded"] });
+        if (!output.pipeline.acquire.attempt_map) {
+            output.pipeline.acquire = buildAcquireStage({ attempts: output.retrieval_method, active: { ok: false, method: output.retrieval_method.at(-1) || "none", content: "", links: [] }, usedBrowser: output.evidence.playwright_attempted, usedArchive: false });
+        }
+        output.pipeline.normalize.status = "partial";
+        output.pipeline.assess = output.assessment;
+        return attachResearchHandoff(output);
     }
-    return promise;
 }
-async function runSmartfetchInner(options, runtime = {}) {
+async function runSmartfetchTask(options, runtime, deadlineAt, output) {
+    // A URL is not a task identity: callers can have different acquisition,
+    // session, runtime and projection requirements. Keep each task independent.
     const requestedTarget = options.target || "auto";
-    const target = inferTarget(options.url, requestedTarget);
-    const timeoutMs = Math.max(3000, Math.floor(Number(options.timeoutMs || 25000)));
-    const output = baseOutput(options.url, target);
+    const target = (requestedTarget === "auto" ? nativeNaverTarget(options.url) : undefined) || inferTarget(options.url, requestedTarget);
+    const remaining = () => remainingFetchTime(deadlineAt);
     output.pipeline.resolve = buildResolveStage({
         url: options.url,
         requestedTarget,
@@ -616,6 +703,7 @@ async function runSmartfetchInner(options, runtime = {}) {
     const profileName = options.profile || configText("fetch.browserProfile");
     const profileDir = profileName ? resolveProfileDir(profileName) : undefined;
     const validation = await validateOutboundUrl(options.url, { allowPrivateHosts: allowPrivateHosts() });
+    remaining();
     if (!validation.ok) {
         return attachResearchHandoff({
             ...output,
@@ -730,16 +818,27 @@ async function runSmartfetchInner(options, runtime = {}) {
         }
     }
     let active;
+    let expectedPostUrl = nativeNaverTarget(options.url) === target ? options.url : undefined;
     if (acquirePlan.lane === "browser") {
-        active = await runPlaywrightFetchWithRuntime(options.url, timeoutMs, target, runtime, profileDir, options.wait_for);
+        const nativeUrl = target === "naver_blog" ? naverBlogMobilePostUrl(options.url)
+            : target === "naver_cafe_post" ? naverCafeMobilePostUrl(options.url) : null;
+        if (nativeUrl && nativeUrl !== options.url) {
+            effectiveReason += `; exact ${target === "naver_blog" ? "Naver Blog" : "Naver Cafe"} native mobile document`;
+            browserAttempt.reason = effectiveReason;
+        }
+        const nativeWait = nativeUrl && target === "naver_cafe_post"
+            ? NAVER_CAFE_READY_SELECTOR
+            : undefined;
+        active = await runPlaywrightFetchWithRuntime(nativeUrl || options.url, remaining(), target, runtime, profileDir, options.wait_for || nativeWait);
         browserAttempt.attempted = true;
         browserAttempt.ok = active.ok;
         browserAttempt.method = active.method;
         output.evidence.playwright_attempted = true;
-        output.evidence.playwright_success = active.ok;
         output.retrieval_method.push(active.method);
         if (active.error)
             output.errors.push(active.error);
+        remaining();
+        output.evidence.playwright_success = active.ok;
     }
     else if (acquirePlan.lane === "seed") {
         active = {
@@ -761,65 +860,156 @@ async function runSmartfetchInner(options, runtime = {}) {
         output.retrieval_method.push(active.method);
     }
     else {
-        active = await runImpitFetch(options.url, timeoutMs);
+        // Leave time for the approved browser when direct transport stalls. Both
+        // phases still share the caller's original deadline.
+        const directDeadline = configText("fetch.browserCdpUrl") && !profileDir
+            ? Math.min(deadlineAt, Date.now() + Math.max(1, Math.floor(remaining() * 0.4)))
+            : deadlineAt;
+        try {
+            active = await withinFetchDeadline(directDeadline, signal => (runtime.runImpitFetch || runImpitFetch)(options.url, directDeadline, signal));
+        }
+        catch (error) {
+            if (!(error instanceof FetchDeadlineError))
+                throw error;
+            active = { ok: false, method: "direct_fetch", content: "", links: [], error: { category: "timeout", code: "direct_acquisition_timeout", message: "Direct acquisition exhausted its share of the original fetch budget" } };
+        }
         directAttempt.attempted = true;
         directAttempt.ok = active.ok;
         directAttempt.method = active.method;
         output.evidence.impit_attempted = true;
-        output.evidence.impit_success = active.ok;
         output.retrieval_method.push(active.method);
         if (active.error)
             output.errors.push(active.error);
+        remaining();
+        output.evidence.impit_success = active.ok;
         const redirectUrl = active.final_url && active.final_url !== options.url ? active.final_url : "";
         const redirectTarget = redirectUrl
-            ? inferTarget(redirectUrl, "auto")
+            ? nativeNaverTarget(redirectUrl) || inferTarget(redirectUrl, "auto")
             : target;
         const redirectProvider = redirectUrl
             ? resolveSmartfetchProvider(redirectUrl, redirectTarget)
             : plannedProvider;
         const needsRedirectBrowserUpgrade = Boolean(redirectUrl)
             && (redirectProvider.capabilities.includes("requires_browser") || redirectProvider.capabilities.includes("interactive_surface"));
-        if (needsRedirectBrowserUpgrade) {
-            const browserResult = await runPlaywrightFetchWithRuntime(redirectUrl, timeoutMs, redirectTarget, runtime, profileDir, options.wait_for);
+        const mustNotRetry = hasTerminalSafetyBlock([active.error]) || active.http_status === 429 || Boolean(active.retry_after);
+        if (mustNotRetry) {
+            browserAttempt.reason = hasTerminalSafetyBlock([active.error])
+                ? "terminal URL safety rejection — no browser continuation"
+                : `direct HTTP ${active.http_status}: cooldown${active.retry_after ? `; Retry-After=${active.retry_after}` : ""} — no browser continuation`;
+        }
+        else if (needsRedirectBrowserUpgrade) {
+            if (!expectedPostUrl && (requestedTarget === "auto" || target === redirectTarget)
+                && (naverBlogMobilePostUrl(redirectUrl) || naverCafeMobilePostUrl(redirectUrl)))
+                expectedPostUrl = redirectUrl;
+            const browserResult = await runPlaywrightFetchWithRuntime(redirectUrl, remaining(), redirectTarget, runtime, profileDir, options.wait_for);
             browserAttempt.attempted = true;
             browserAttempt.ok = browserResult.ok;
             browserAttempt.method = browserResult.method;
             browserAttempt.reason = `redirected final provider:${redirectProvider.id} requires browser retrieval`;
             output.evidence.playwright_attempted = true;
-            output.evidence.playwright_success = browserResult.ok;
             output.retrieval_method.push(browserResult.method);
             if (browserResult.error)
                 output.errors.push(browserResult.error);
+            remaining();
+            output.evidence.playwright_success = browserResult.ok;
             effectiveLane = "browser";
             effectiveReason = `redirected final provider:${redirectProvider.id} requires browser retrieval`;
             redirectUpgrade = true;
-            if (browserResult.ok)
+            if (browserResult.ok || expectedPostUrl || hasTerminalSafetyBlock([browserResult.error]))
                 active = browserResult;
         }
-        else if (needsBotChallengeBrowserUpgrade(active)) {
-            // impit returned a Cloudflare/bot-challenge interstitial — try the browser lane.
-            const browserResult = await runPlaywrightFetchWithRuntime(options.url, timeoutMs, target, runtime, profileDir, options.wait_for);
+        else if (operatorRecoveryReason(active) || needsBotChallengeBrowserUpgrade(active)) {
+            // Direct-client failure does not prove the approved browser cannot read it.
+            const browserResult = await runPlaywrightFetchWithRuntime(options.url, remaining(), target, runtime, profileDir, options.wait_for);
             browserAttempt.attempted = true;
             browserAttempt.ok = browserResult.ok;
             browserAttempt.method = browserResult.method;
-            browserAttempt.reason = "impit returned bot-challenge interstitial — upgraded to browser";
+            browserAttempt.reason = operatorRecoveryReason(active) || "impit returned bot-challenge interstitial — upgraded to browser";
             output.evidence.playwright_attempted = true;
-            output.evidence.playwright_success = browserResult.ok;
             output.retrieval_method.push(browserResult.method);
             if (browserResult.error)
                 output.errors.push(browserResult.error);
+            remaining();
+            output.evidence.playwright_success = browserResult.ok;
             effectiveLane = "browser";
-            effectiveReason = "impit returned bot-challenge interstitial — upgraded to browser";
+            effectiveReason = browserAttempt.reason;
             if (browserResult.ok)
                 active = browserResult;
         }
     }
+    // Browser-resolved aliases must bind the observed article before continuing.
+    if (!expectedPostUrl && active.ok && active.final_url && !hasTerminalSafetyBlock([active.error]) && active.http_status !== 429 && !active.retry_after) {
+        const landedTarget = nativeNaverTarget(active.final_url);
+        if (landedTarget && (requestedTarget === "auto" || target === landedTarget)) {
+            expectedPostUrl = active.final_url;
+            if (!["m.blog.naver.com", "m.cafe.naver.com"].includes(new URL(active.final_url).hostname)) {
+                const browserResult = await runPlaywrightFetchWithRuntime(expectedPostUrl, remaining(), landedTarget, runtime, profileDir, options.wait_for);
+                browserAttempt.attempted = true;
+                browserAttempt.ok = browserResult.ok;
+                browserAttempt.method = browserResult.method;
+                browserAttempt.reason = "validated alias resolved to an exact Naver native article";
+                output.evidence.playwright_attempted = true;
+                output.retrieval_method.push(browserResult.method);
+                if (browserResult.error)
+                    output.errors.push(browserResult.error);
+                remaining();
+                output.evidence.playwright_success = browserResult.ok;
+                effectiveLane = "browser";
+                effectiveReason = browserAttempt.reason;
+                redirectUpgrade = true;
+                active = browserResult;
+            }
+        }
+    }
     const archiveTarget = inferTarget(active.final_url || options.url, "auto");
     const archiveProvider = resolveSmartfetchProvider(active.final_url || options.url, archiveTarget);
+    const suppressRetries = hasTerminalSafetyBlock([active.error]) || active.http_status === 429 || Boolean(active.retry_after);
+    // A visible review-to-unlock prompt is a participation boundary, not a
+    // weak article to be sent through third-party relay/archive retries.
+    const lockedBlindPreview = ["teamblind.com", "www.teamblind.com"].includes(new URL(options.url).hostname)
+        && /^\/kr\/company\/[^/]+\/review\/[^/]+\/?$/.test(new URL(options.url).pathname)
+        && /class=["'][^"']*\breview_unlock\b/.test(active.content);
+    // NamuWiki's weak anonymous shell can consume the entire deadline in the
+    // reader relay even when the approved local browser can read the original.
+    // Keep the direct result and existing relay fallback if browser recovery fails.
+    if (effectiveLane === "direct" && target === "namuwiki_article" && configText("fetch.browserCdpUrl")
+        && !profileDir && !suppressRetries && needsDynamicCrawl(active) && !browserAttempt.attempted) {
+        const browserResult = await runPlaywrightFetchWithRuntime(options.url, remaining(), target, runtime, profileDir, options.wait_for);
+        const original = new URL(options.url);
+        let landed = null;
+        try {
+            if (browserResult.final_url)
+                landed = new URL(browserResult.final_url);
+        }
+        catch { /* Unusable browser identity. */ }
+        // A full browser DOM may contain unrelated CAPTCHA/help terms in chrome;
+        // use the isolated primary content when available, not a page-wide token.
+        const usable = browserResult.ok && ((browserResult.primary_content?.text?.length || 0) >= 200 || !needsDynamicCrawl(browserResult))
+            && landed?.origin === original.origin && landed.pathname === original.pathname && landed.search === original.search;
+        browserAttempt.attempted = true;
+        browserAttempt.ok = Boolean(usable);
+        browserAttempt.method = browserResult.method;
+        browserAttempt.reason = "weak NamuWiki direct document — approved browser before optional relays";
+        output.evidence.playwright_attempted = true;
+        output.retrieval_method.push(browserResult.method);
+        if (browserResult.error)
+            output.errors.push(browserResult.error);
+        remaining();
+        output.evidence.playwright_success = Boolean(usable);
+        if (usable) {
+            active = browserResult;
+            effectiveLane = "browser";
+            effectiveReason = browserAttempt.reason;
+        }
+    }
     const allowArchiveFallback = acquirePlan.allowArchiveFallback
+        && !suppressRetries
+        && !lockedBlindPreview
         && effectiveLane === "direct"
         && Boolean(archiveProvider.policy.allowArchiveFallback);
     const allowJinaReader = acquirePlan.allowJinaReader
+        && !suppressRetries
+        && !lockedBlindPreview
         && effectiveLane === "direct"
         && Boolean(archiveProvider.policy.allowJinaReader);
     if (!allowJinaReader) {
@@ -832,9 +1022,10 @@ async function runSmartfetchInner(options, runtime = {}) {
             ? "archive fallback was not eligible for the effective provider/lane"
             : "archive fallback was not allowed by plan, privacy profile, or settings";
     }
-    if (allowJinaReader) {
+    if (allowJinaReader && effectiveLane === "direct") {
         const beforeMethod = active.method;
-        const relayed = await maybeUseJinaReader(active.final_url || options.url, active, timeoutMs);
+        const relayed = await maybeUseJinaReader(active.final_url || options.url, active, remaining());
+        remaining();
         active = relayed.active;
         output.errors.push(...relayed.errors);
         const jinaErrors = relayed.errors.filter((error) => error.code.startsWith("jina_reader"));
@@ -853,9 +1044,10 @@ async function runSmartfetchInner(options, runtime = {}) {
         if (jinaSucceeded)
             output.retrieval_method.push(active.method);
     }
-    if (allowArchiveFallback) {
+    if (allowArchiveFallback && effectiveLane === "direct") {
         const beforeMethod = active.method;
-        const archived = await maybeUseArchiveFallback(active.final_url || options.url, active, timeoutMs);
+        const archived = await maybeUseArchiveFallback(active.final_url || options.url, active, remaining());
+        remaining();
         active = archived.active;
         output.errors.push(...archived.errors);
         const archiveErrors = archived.errors.filter((error) => /archive|wayback/i.test(error.code));
@@ -877,7 +1069,8 @@ async function runSmartfetchInner(options, runtime = {}) {
         }
     }
     if (effectiveLane === "direct" && (needsDynamicCrawl(active) || needsSpaBrowserFallback(active))) {
-        const scraped = await maybeUseAdaptiveScraper(active.final_url || options.url, active, timeoutMs);
+        const scraped = await maybeUseAdaptiveScraper(active.final_url || options.url, active, remaining());
+        remaining();
         output.errors.push(...scraped.errors);
         adaptiveScraperAttempt.attempted = scraped.attempted;
         adaptiveScraperAttempt.ok = scraped.succeeded;
@@ -900,24 +1093,26 @@ async function runSmartfetchInner(options, runtime = {}) {
     // SPA auto-fallback: after cheap alternatives (Jina, archive, adaptive scraper) didn't help,
     // if the direct fetch result looks like an SPA shell, upgrade to Playwright.
     // This must come AFTER Jina/archive so those cheap paths get a chance first.
-    if (effectiveLane === "direct" && needsSpaBrowserFallback(active)) {
-        const browserResult = await runPlaywrightFetchWithRuntime(options.url, timeoutMs, target, runtime, profileDir, options.wait_for);
+    if (effectiveLane === "direct" && !browserAttempt.attempted && needsSpaBrowserFallback(active)) {
+        const browserResult = await runPlaywrightFetchWithRuntime(options.url, remaining(), target, runtime, profileDir, options.wait_for);
         browserAttempt.attempted = true;
         browserAttempt.ok = browserResult.ok;
         browserAttempt.method = browserResult.method;
         browserAttempt.reason = "direct fetch returned SPA shell — auto-fallback to browser for JS rendering";
         output.evidence.playwright_attempted = true;
-        output.evidence.playwright_success = browserResult.ok;
         output.retrieval_method.push(browserResult.method);
         if (browserResult.error)
             output.errors.push(browserResult.error);
+        remaining();
+        output.evidence.playwright_success = browserResult.ok;
         effectiveLane = "browser";
         effectiveReason = "direct fetch returned SPA shell — auto-fallback to browser for JS rendering";
         if (browserResult.ok)
             active = browserResult;
     }
-    let effectiveTarget = target;
-    if ((options.target || "auto") === "auto") {
+    let effectiveTarget = (expectedPostUrl ? nativeNaverTarget(expectedPostUrl) : undefined) || target;
+    // The identity parser and selected normalizer must agree on accepted URLs.
+    if ((options.target || "auto") === "auto" && !expectedPostUrl) {
         const inferred = inferTarget(active.final_url || options.url, "auto");
         if (inferred !== effectiveTarget) {
             effectiveTarget = inferred;
@@ -925,6 +1120,8 @@ async function runSmartfetchInner(options, runtime = {}) {
             output.risk_level = riskLevelForTarget(inferred);
         }
     }
+    output.target = effectiveTarget;
+    output.risk_level = riskLevelForTarget(effectiveTarget);
     output.pipeline.resolve = buildResolveStage({
         url: options.url,
         requestedTarget,
@@ -953,17 +1150,39 @@ async function runSmartfetchInner(options, runtime = {}) {
         ],
     };
     const providerUrl = active.final_url || options.url;
-    let provider = resolveSmartfetchProvider(providerUrl, effectiveTarget);
+    const exactNaverProviderUrl = expectedPostUrl || (target === "naver_blog" || target === "naver_cafe_post" ? options.url : undefined);
+    let provider = resolveSmartfetchProvider(exactNaverProviderUrl || providerUrl, effectiveTarget);
     if (provider.id === "generic" && providerUrl !== options.url) {
         const originalProvider = resolveSmartfetchProvider(options.url, effectiveTarget);
         if (originalProvider.id !== "generic")
             provider = originalProvider;
     }
+    if (suppressRetries) {
+        // Normalizers can perform their own feed/API/search I/O. Do not let them
+        // silently repeat a cooldown request or reopen a terminal safety rejection.
+        output.partial = true;
+        output.assessment = createAssessment({
+            confidence: "low", blocked_reason: active.error?.message || "Retrieval stopped",
+            signals: [hasTerminalSafetyBlock([active.error]) ? "resolve:blocked" : "retrieval:cooldown"],
+        });
+        output.pipeline.normalize = buildNormalizeStage({ provider, partial: true });
+        output.pipeline.assess = output.assessment;
+        return attachResearchHandoff(output);
+    }
     const normalized = await provider.normalize({
         url: options.url,
         resolvedUrl: providerUrl,
+        ...(expectedPostUrl ? { expectedPostUrl } : {}),
         target: effectiveTarget,
-        timeoutMs,
+        timeoutMs: remaining(),
+        deadlineAt,
+        browserSessionOptions: {
+            ...(!profileDir && configText("fetch.browserCdpUrl") ? { cdpUrl: configText("fetch.browserCdpUrl") } : {}),
+            ...(profileDir ? { profileDir } : {}),
+            allowPrivateHosts: allowPrivateHosts(),
+            waitUntil: choosePlaywrightWaitUntil(effectiveTarget, options.url),
+            extraHeaders: browserFetchHeaders(options.url),
+        },
         active,
         errors: output.errors,
         runBrowserSession: runtime.runBrowserSession || runBrowserSession,
@@ -972,15 +1191,17 @@ async function runSmartfetchInner(options, runtime = {}) {
         runCoupangSearch: runtime.runCoupangSearch || runCoupang,
         ...(options.includeMarkdown ? { includeMarkdown: true } : {}),
     });
+    remaining();
     const academic = await maybeUseAcademicFallback({
         url: options.url,
         resolvedUrl: providerUrl,
         target: effectiveTarget,
-        timeoutMs,
+        timeoutMs: remaining(),
         active,
         errors: output.errors,
         ...(runtime.runSmartsearch ? { runSmartsearch: runtime.runSmartsearch } : {}),
     }, normalized);
+    remaining();
     const enriched = academic.normalized;
     output.retrieval_method.push(normalized.method);
     output.retrieval_method.push(...academic.retrievalMethods);
@@ -990,7 +1211,7 @@ async function runSmartfetchInner(options, runtime = {}) {
     output.outbound_links = Array.isArray(enriched.outbound_links) ? enriched.outbound_links : [];
     output.partial = enriched.partial;
     output.errors = enriched.errors;
-    output.assets = discoverAssets(options.url, active.content, output.post, output.outbound_links);
+    output.assets = discoverAssets(options.url, normalized.assetHtml ?? active.content, output.post, output.outbound_links, active.method === "wayback_archive");
     if (isNaverCafeUnavailableFallback(provider.id, output) || isNaverMapSearchUnavailable(provider.id, output) || isThreadsJoinShellUnavailable(provider.id, output) || isInstagramUnavailable(provider.id, output) || isDcinsideUnavailable(provider.id, output) || isTelegramUnavailable(provider.id, output) || isXUnavailable(provider.id, output)) {
         output.assets = [];
     }
@@ -1004,6 +1225,24 @@ async function runSmartfetchInner(options, runtime = {}) {
         output.assets = [];
         output.download = null;
     }
+    if (options.exportComments) {
+        try {
+            output.comment_export = exportComments(output);
+        }
+        catch {
+            output.partial = true;
+            output.errors.push({ category: "unavailable", code: "comment_export_failed", message: "Private comment export failed or exceeded its 5000-row/8 MiB bound; no export artifact is claimed" });
+        }
+    }
+    if (options.exportBody) {
+        try {
+            output.body_export = exportBody(output);
+        }
+        catch {
+            output.partial = true;
+            output.errors.push({ category: "unavailable", code: "body_export_failed", message: "Private body export failed, had no readable body, or exceeded its 1 MiB bound; no export artifact is claimed" });
+        }
+    }
     output.pipeline.normalize = buildNormalizeStage({ provider, partial: output.partial });
     output.assessment = assessSmartfetchOutput({
         url: options.url,
@@ -1063,13 +1302,35 @@ async function runSmartfetchInner(options, runtime = {}) {
     }
     return projectedOutput;
 }
+function renderCommentBody(comment) {
+    if (!comment)
+        return "";
+    const images = Array.isArray(comment.image_urls) ? comment.image_urls.filter((url) => typeof url === "string") : [];
+    return [asString(comment.body) || asString(comment.text), ...images].filter(Boolean).join("\n").trim();
+}
 export function renderSmartfetch(output, format) {
     const projected = output.budget ? output : projectSmartfetchOutput(output);
     const assessment = projected.assessment || createAssessment();
     const post = projected.post || {};
     const title = String(post?.title || "").trim();
     const text = String(post?.text || post?.selftext || "").trim();
+    const exported = projected.comment_export;
+    const exportReceipt = exported ? `Comment export: ${exported.path}\ncomments=${exported.comments_count} complete=${exported.complete} bytes=${exported.bytes} sha256=${exported.sha256}` : "";
+    const bodyExported = projected.body_export;
+    const bodyExportReceipt = bodyExported ? `Body export: ${bodyExported.path}\nchars=${bodyExported.chars} complete=${bodyExported.complete} bytes=${bodyExported.bytes} sha256=${bodyExported.sha256}` : "";
     if (format === "text") {
+        // Explicit full output must not hide projected comments behind the compact text view.
+        // The projection policy still bounds each field and the total response.
+        const fullPolicy = projected.budget?.policy === "smartfetch.full" ? resolveSmartfetchBudgetPolicy("full") : null;
+        const textLimits = fullPolicy ? {
+            ...TEXT_RENDER_LIMITS,
+            bodyChars: fullPolicy.maxPostTextChars,
+            itemChars: fullPolicy.maxItemTextChars,
+            results: fullPolicy.maxThreadItems,
+            comments: fullPolicy.maxCommentItems,
+            links: fullPolicy.maxOutboundLinks,
+            assets: fullPolicy.maxAssets,
+        } : TEXT_RENDER_LIMITS;
         const assessmentLines = [
             `confidence=${assessment.confidence}`,
             assessment.blocked_reason ? `blocked=${assessment.blocked_reason}` : "",
@@ -1086,12 +1347,13 @@ export function renderSmartfetch(output, format) {
             post?.availability ? `availability=${post.availability}` : "",
             assessmentLines,
         ].filter(Boolean).join(" ");
-        const resultItems = Array.isArray(projected.thread) ? limitedItems(projected.thread, TEXT_RENDER_LIMITS.results) : { visible: [], omitted: 0 };
-        const commentItems = Array.isArray(projected.comments) ? limitedItems(projected.comments, TEXT_RENDER_LIMITS.comments) : { visible: [], omitted: 0 };
-        const hasHandoff = Boolean(assessment.recommended_handoff || assessment.auth_required_likely || assessment.interactive_flow_likely);
-        const linkItems = limitedItems(cleanLinks(projected.outbound_links || []), hasHandoff ? TEXT_RENDER_LIMITS.handoffLinks : TEXT_RENDER_LIMITS.links);
-        const assetItems = Array.isArray(projected.assets) ? limitedItems(projected.assets, TEXT_RENDER_LIMITS.assets) : { visible: [], omitted: 0 };
-        const textBody = compactText(text, hasHandoff ? TEXT_RENDER_LIMITS.handoffBodyChars : TEXT_RENDER_LIMITS.bodyChars);
+        const resultItems = Array.isArray(projected.thread) ? limitedItems(projected.thread, textLimits.results) : { visible: [], omitted: 0 };
+        const commentItems = Array.isArray(projected.comments) ? limitedItems(projected.comments, textLimits.comments) : { visible: [], omitted: 0 };
+        const hasHandoff = Boolean(assessment.recommended_handoff || assessment.auth_required_likely || assessment.interactive_flow_likely
+            || (assessment.continuation && (assessment.continuation.status !== "unavailable" || assessment.confidence === "low")));
+        const linkItems = limitedItems(cleanLinks(projected.outbound_links || []), hasHandoff ? TEXT_RENDER_LIMITS.handoffLinks : textLimits.links);
+        const assetItems = Array.isArray(projected.assets) ? limitedItems(projected.assets, textLimits.assets) : { visible: [], omitted: 0 };
+        const textBody = compactText(text, hasHandoff ? TEXT_RENDER_LIMITS.handoffBodyChars : textLimits.bodyChars);
         const renderBudget = [
             textBody.truncated ? `text_chars_omitted=${textBody.omittedChars}` : "",
             resultItems.omitted ? `results_omitted=${resultItems.omitted}` : "",
@@ -1103,21 +1365,22 @@ export function renderSmartfetch(output, format) {
             const title = String(item?.title || "").trim();
             const price = String(item?.price || "").trim();
             const url = String(item?.url || "").trim();
-            return compactText([title, price, url].filter(Boolean).join(" — "), TEXT_RENDER_LIMITS.itemChars).text;
+            return compactText([title, price, url].filter(Boolean).join(" — "), textLimits.itemChars).text;
         }).filter(Boolean);
         const comments = commentItems.visible.map((comment) => {
-            const body = String(comment?.body || "").trim();
+            const body = renderCommentBody(comment);
             const author = String(comment?.author || "unknown");
-            return body ? compactText(`${author}: ${body}`, TEXT_RENDER_LIMITS.itemChars).text : "";
+            return body ? compactText(`${author}: ${body}`, textLimits.itemChars).text : "";
         }).filter(Boolean);
         const assets = assetItems.visible.map((asset) => `${asset.kind}: ${asset.download_url}`);
         const download = projected.download ? `download: ${projected.download.download_url}` : "";
         const handoff = assessment.recommended_handoff
-            ? `handoff: ${assessment.recommended_handoff.tool} — ${assessment.recommended_handoff.reason}${assessment.recommended_handoff.captcha_type ? ` [${assessment.recommended_handoff.captcha_type}]` : ""}`
+            ? `handoff: ${assessment.recommended_handoff.tool} — ${assessment.recommended_handoff.reason}`
             : "";
+        const continuation = assessment.continuation ? `continuation: ${assessment.continuation.status} — ${assessment.continuation.reason}` : "";
         const budget = projected.budget?.truncated ? `budget: truncated fields=${Object.keys(projected.budget.fields).join(",")}` : "";
-        const compactBudget = renderBudget ? `text_view: compact renderer omitted ${renderBudget}` : "";
-        return [title, meta, textBody.text, results.join("\n"), comments.join("\n"), linkItems.visible.join("\n"), download, assets.join("\n"), handoff, budget, compactBudget].filter(Boolean).join("\n\n").trim();
+        const compactBudget = renderBudget ? `text_view: ${fullPolicy ? "full" : "compact"} renderer omitted ${renderBudget}` : "";
+        return [title, meta, exportReceipt, bodyExportReceipt, textBody.text, results.join("\n"), comments.join("\n"), linkItems.visible.join("\n"), download, assets.join("\n"), handoff, continuation, budget, compactBudget].filter(Boolean).join("\n\n").trim();
     }
     const parts = [];
     const url = String(post?.url || projected.url || "").trim();
@@ -1140,6 +1403,10 @@ export function renderSmartfetch(output, format) {
         parts.push(`# ${title}`);
     if (url)
         parts.push(`Source: ${url}`);
+    if (exportReceipt)
+        parts.push(exportReceipt);
+    if (bodyExportReceipt)
+        parts.push(bodyExportReceipt);
     parts.push(`Confidence: ${assessment.confidence}`);
     if (assessment.blocked_reason)
         parts.push(`Blocked reason: ${assessment.blocked_reason}`);
@@ -1154,7 +1421,7 @@ export function renderSmartfetch(output, format) {
     if (Array.isArray(projected.comments) && projected.comments.length > 0) {
         const comments = projected.comments.map((comment) => {
             const author = String(comment?.author || "unknown");
-            const body = String(comment?.body || "").trim();
+            const body = renderCommentBody(comment);
             return body ? `- ${author}: ${body}` : "";
         }).filter(Boolean);
         if (comments.length > 0)
@@ -1189,8 +1456,11 @@ export function renderSmartfetch(output, format) {
         parts.push(`## Assets\n\n${assets.join("\n")}`);
     if (assessment.recommended_handoff) {
         const handoff = assessment.recommended_handoff;
-        const captchaLine = handoff.captcha_type ? `\n- captcha_type: ${handoff.captcha_type}` : "";
-        parts.push(`## Recommended handoff\n\n- tool: ${handoff.tool}\n- mode: ${handoff.mode}\n- url: ${handoff.url}\n- reason: ${handoff.reason}\n- goal: ${handoff.goal}${captchaLine}`);
+        parts.push(`## Recommended handoff\n\n- tool: ${handoff.tool}\n- mode: ${handoff.mode}\n- url: ${handoff.url}\n- reason: ${handoff.reason}\n- goal: ${handoff.goal}`);
+    }
+    if (assessment.continuation) {
+        const continuation = assessment.continuation;
+        parts.push(`## Continuation\n\n- status: ${continuation.status}\n- reason: ${continuation.reason}\n- url: ${continuation.url}`);
     }
     if (projected.budget?.truncated) {
         const fields = Object.entries(projected.budget.fields).map(([field, item]) => {
@@ -1215,12 +1485,19 @@ export async function renderSmartfetchUrl(url, format, timeoutMs) {
     return renderSmartfetch(output, format);
 }
 async function runPlaywrightFetchWithRuntime(url, timeoutMs, target, runtime, profileDir, waitForSelector) {
+    const nativeUrl = target === "naver_blog" ? naverBlogMobilePostUrl(url) : target === "naver_cafe_post" ? naverCafeMobilePostUrl(url) : null;
+    if (nativeUrl) {
+        url = nativeUrl;
+        if (!waitForSelector && target === "naver_cafe_post")
+            waitForSelector = NAVER_CAFE_READY_SELECTOR;
+    }
     if (!runtime.runBrowserSession)
         return runPlaywrightFetch(url, timeoutMs, target, profileDir, waitForSelector);
     const extraHeaders = browserFetchHeaders(url);
     const result = await runtime.runBrowserSession({
         url,
         timeoutMs,
+        ...(!profileDir && configText("fetch.browserCdpUrl") ? { cdpUrl: configText("fetch.browserCdpUrl") } : {}),
         allowPrivateHosts: allowPrivateHosts(),
         waitUntil: choosePlaywrightWaitUntil(target, url),
         extraHeaders,