smart-web-mcp 0.4.3 → 0.5.0

package/CHANGELOG.md

@@ -6,6 +6,37 @@ The format is based on Keep a Changelog and this project uses SemVer.
 
 ## [Unreleased]
 
+## [0.5.0] - 2026-03-23
+
+### Added
+
+- `smartcrawl` for local browser-driven exploration of boards, docs sites, pagination, and scroll-heavy pages
+- shared stealth browser session runtime used by both `smartcrawl` and `smartfetch`
+- npm-managed `pre-commit` and `pre-push` hook installation for local TypeScript and full repo checks
+
+### Changed
+
+- configuration UX now centers on `SMART_WEB_PROFILE` with fine-grained env flags treated as advanced overrides
+- README, examples, and getting-started docs now use one npm-first path instead of parallel npm/Bun setup tracks
+
+### Fixed
+
+- MCP registry metadata now keeps the advanced environment-variable overrides that the runtime still supports
+- Playwright fetch now preserves HTML content for provider normalization instead of dropping back to plain text first
+- DCInside parsing now handles recommendation boxes with additional classes
+
+## [0.4.4] - 2026-03-21
+
+### Added
+
+- `smartfetch.assets` structured output with discovered media and attachment download URLs
+- attachment/media rendering in markdown and text output
+
+### Fixed
+
+- provider fetch tests now cover redirect-to-private-host blocking
+- YouTube results expose thumbnail-style asset metadata for downstream download flows
+
 ## [0.4.3] - 2026-03-21
 
 ### Added

package/README.md

@@ -2,9 +2,13 @@
 
 # smart-web
 
-A web MCP that works.
+One local MCP server for web search, page fetch, and site crawl.
 
-`smart-web` bundles `smartsearch` and `smartfetch` into one local MCP server for Claude Code, OpenCode, and other MCP clients.
+`smart-web` ships three tools in one stdio server:
+
+- `smartsearch`: fallback web search with optional site-crawl handoff
+- `smartfetch`: browser-aware fetch with target-specific normalization
+- `smartcrawl`: site exploration for docs, boards, pagination, and scroll-heavy pages
 
 - npm: `https://www.npmjs.com/package/smart-web-mcp`
 - MCP Registry: `io.github.rich-jojo/smart-web`
@@ -16,11 +20,7 @@ A web MCP that works.
 npx -y smart-web-mcp
 ```
 
-If your team uses Bun:
-
-```bash
-bunx smart-web-mcp
-```
+`npm` is the canonical package-manager path for this repo and package.
 
 ## Quick start
 
@@ -45,28 +45,48 @@ OpenCode:
 }
 ```
 
-## What it includes
+## Configuration
+
+Most users only need these variables:
+
+- `SMART_WEB_PROFILE=balanced`: default behavior
+- `SMART_WEB_PROFILE=private`: disables relay-style providers and public search helpers by default
+- `SEARXNG_BASE_URL=https://search.example.com`: self-hosted private fallback search
+- `EXA_API_KEY` and `BRAVE_SEARCH_API_KEY`: optional paid search tiers
+
+Everything else is still supported, but treated as an advanced override rather than the main setup path.
 
-- `smartsearch`: fallback web search with Exa, Brave, SearXNG, DuckDuckGo, and Brave HTML
-- `smartsearch`: plus direct site-native fallbacks for Velog and StackExchange-style sites when that is more reliable
-- `smartfetch`: browser-aware fetch with special handling for Reddit, X, YouTube, DCInside, and Naver Blog
-- sensible safety defaults: private-network blocking, local-only mode, provider toggles
+## Smoke test
+
+- ask `smartsearch` for `site:velog.io react useeffect cleanup`
+- ask `smartfetch` to fetch a normal article URL
+- ask `smartcrawl` to explore a docs site or forum you actually use
 
 ## Docs
 
-- `docs/getting-started.md`
-- `docs/configuration.md`
-- `docs/providers.md`
-- `docs/architecture.md`
-- `docs/composition.md`
-- `docs/releasing.md`
-- `CHANGELOG.md`
+- [Getting started](docs/getting-started.md)
+- [Configuration](docs/configuration.md)
+- [Crawling](docs/crawling.md)
+- [Providers](docs/providers.md)
+- [Architecture](docs/architecture.md)
+- [Contributing](CONTRIBUTING.md)
+- [CHANGELOG](CHANGELOG.md)
+
+## Examples
+
+- [Claude Code](examples/claude.mcp.json)
+- [OpenCode](examples/opencode.json)
+
+These examples use `/absolute/path/to/...` placeholders for local checkouts.
 
-## Status
+## Development
+
+```bash
+npm install
+npm run check
+```
 
-- npm package is published
-- MCP Registry metadata is published
-- GitHub Actions verify package, registry metadata, and MCP wiring
+`npm install` also installs local git hooks for fast TypeScript checks on commit and the full verification suite on push.
 
 ## License
 

package/dist/browser-session.d.ts

@@ -0,0 +1,41 @@
+import type { ToolError } from "./shared.js";
+type BrowserWaitUntil = "domcontentloaded" | "networkidle";
+export type BrowserAnchor = {
+    href: string;
+    text: string;
+    context: string;
+    rel: string;
+    class_name: string;
+    aria_label: string;
+};
+export type BrowserPageSnapshot = {
+    ok: true;
+    method: "playwright_stealth";
+    url: string;
+    final_url: string;
+    title: string;
+    html: string;
+    text: string;
+    links: string[];
+    anchors: BrowserAnchor[];
+    scroll_steps: number;
+    load_more_clicks: number;
+};
+type BrowserPageFailure = {
+    ok: false;
+    method: "playwright_stealth";
+    error: ToolError;
+};
+export type BrowserPageResult = BrowserPageSnapshot | BrowserPageFailure;
+export type BrowserSessionOptions = {
+    url: string;
+    timeoutMs: number;
+    allowPrivateHosts?: boolean;
+    waitUntil?: BrowserWaitUntil;
+    extraHeaders?: Record<string, string>;
+    scrollSteps?: number;
+    maxLoadMoreClicks?: number;
+    maxAnchors?: number;
+};
+export declare function runBrowserSession(options: BrowserSessionOptions): Promise<BrowserPageResult>;
+export {};

package/dist/browser-session.js

@@ -0,0 +1,206 @@
+import { asString, dedupeUrls, resolveValidatedUrl } from "./shared.js";
+const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36";
+let chromiumCache = undefined;
+async function getChromium() {
+    if (chromiumCache !== undefined)
+        return chromiumCache;
+    try {
+        const playwrightExtra = await import("playwright-extra");
+        const stealthModule = await import("puppeteer-extra-plugin-stealth");
+        const chromium = playwrightExtra.chromium;
+        const createStealthPlugin = stealthModule.default || stealthModule;
+        chromium.use(createStealthPlugin());
+        chromiumCache = chromium;
+    }
+    catch {
+        chromiumCache = false;
+    }
+    return chromiumCache;
+}
+async function launchBrowser(chromium) {
+    const attempts = [
+        { channel: process.env.SMARTFETCH_CHROME_CHANNEL || "chrome", headless: true },
+        { executablePath: process.env.SMARTFETCH_CHROME_PATH, headless: true },
+        { headless: true },
+    ];
+    let lastError = null;
+    for (const options of attempts) {
+        const launchOptions = { ...options };
+        if (launchOptions.executablePath === undefined)
+            delete launchOptions.executablePath;
+        try {
+            return await chromium.launch(launchOptions);
+        }
+        catch (error) {
+            lastError = error;
+        }
+    }
+    throw lastError;
+}
+async function clickLoadMore(page) {
+    return page.evaluate(() => {
+        const pattern = /^(load more|show more|more posts|more results|view more|see more|더보기|더 보기|더 불러오기)$/i;
+        const nodes = Array.from(document.querySelectorAll("button, [role='button'], input[type='button'], input[type='submit'], a"));
+        for (const node of nodes) {
+            const element = node;
+            const text = (element.innerText || element.getAttribute("aria-label") || element.getAttribute("title") || "").replace(/\s+/g, " ").trim();
+            if (!text || !pattern.test(text))
+                continue;
+            if (node instanceof HTMLAnchorElement) {
+                const href = node.getAttribute("href") || "";
+                if (href && href !== "#" && !href.toLowerCase().startsWith("javascript:"))
+                    continue;
+            }
+            const rect = element.getBoundingClientRect();
+            if (rect.width <= 0 || rect.height <= 0)
+                continue;
+            element.click();
+            return true;
+        }
+        return false;
+    });
+}
+async function drivePage(page, options) {
+    const scrollSteps = Math.max(0, Math.min(20, Math.floor(Number(options.scrollSteps || 0))));
+    const maxLoadMoreClicks = Math.max(0, Math.min(10, Math.floor(Number(options.maxLoadMoreClicks || 0))));
+    let completedScrollSteps = 0;
+    let loadMoreClicks = 0;
+    let stagnantRounds = 0;
+    let previousAnchorCount = await page.locator("a[href]").count().catch(() => 0);
+    let previousHeight = await page.evaluate(() => document.body?.scrollHeight || document.documentElement?.scrollHeight || 0);
+    for (let step = 0; step < scrollSteps; step++) {
+        let clicked = false;
+        if (loadMoreClicks < maxLoadMoreClicks) {
+            clicked = await clickLoadMore(page).catch(() => false);
+            if (clicked) {
+                loadMoreClicks += 1;
+                await page.waitForTimeout(700);
+            }
+        }
+        await page.evaluate(() => window.scrollTo(0, document.body?.scrollHeight || document.documentElement?.scrollHeight || 0));
+        completedScrollSteps += 1;
+        await page.waitForTimeout(700);
+        await page.waitForLoadState("networkidle", { timeout: 1500 }).catch(() => { });
+        const currentAnchorCount = await page.locator("a[href]").count().catch(() => previousAnchorCount);
+        const currentHeight = await page.evaluate(() => document.body?.scrollHeight || document.documentElement?.scrollHeight || 0);
+        if (!clicked && currentAnchorCount <= previousAnchorCount && currentHeight <= previousHeight) {
+            stagnantRounds += 1;
+        }
+        else {
+            stagnantRounds = 0;
+        }
+        previousAnchorCount = currentAnchorCount;
+        previousHeight = currentHeight;
+        if (stagnantRounds >= 2)
+            break;
+    }
+    return { scroll_steps: completedScrollSteps, load_more_clicks: loadMoreClicks };
+}
+async function extractAnchors(page, maxAnchors) {
+    return page.evaluate((maxItems) => {
+        const nodes = Array.from(document.querySelectorAll("a[href]")).slice(0, maxItems);
+        return nodes.map((node) => {
+            const element = node;
+            const closest = element.closest("article, li, tr, section, main, .topic-list-item, .topic-body, .topic-post, .athing, .titleline, [role='article'], .feed-item, .timeline-item, .post, .storylink");
+            const text = (element.innerText || element.getAttribute("aria-label") || element.getAttribute("title") || "").replace(/\s+/g, " ").trim();
+            const contextSource = closest?.innerText || element.innerText || element.textContent || "";
+            return {
+                href: element.href || "",
+                text,
+                context: contextSource.replace(/\s+/g, " ").trim().slice(0, 500),
+                rel: element.rel || "",
+                class_name: typeof element.className === "string" ? element.className : "",
+                aria_label: element.getAttribute("aria-label") || "",
+            };
+        });
+    }, maxAnchors);
+}
+export async function runBrowserSession(options) {
+    const chromium = await getChromium();
+    if (!chromium) {
+        return {
+            ok: false,
+            method: "playwright_stealth",
+            error: {
+                category: "unavailable",
+                code: "playwright_unavailable",
+                message: "Playwright browser runtime is unavailable",
+            },
+        };
+    }
+    const resolveOptions = {
+        timeoutMs: options.timeoutMs,
+    };
+    if (options.allowPrivateHosts !== undefined)
+        resolveOptions.allowPrivateHosts = options.allowPrivateHosts;
+    if (options.extraHeaders !== undefined)
+        resolveOptions.headers = options.extraHeaders;
+    const resolved = await resolveValidatedUrl(options.url, resolveOptions);
+    if (!resolved.ok) {
+        return {
+            ok: false,
+            method: "playwright_stealth",
+            error: {
+                category: "block",
+                code: resolved.reason,
+                message: resolved.message,
+            },
+        };
+    }
+    let browser = null;
+    try {
+        browser = await launchBrowser(chromium);
+        const context = await browser.newContext({
+            userAgent: DEFAULT_USER_AGENT,
+            locale: "ko-KR",
+            extraHTTPHeaders: options.extraHeaders,
+        });
+        const page = await context.newPage();
+        await page.goto(resolved.url, {
+            waitUntil: options.waitUntil || "networkidle",
+            timeout: options.timeoutMs,
+        });
+        const driven = await drivePage(page, options);
+        const html = await page.content();
+        const text = await page.evaluate(() => document.body?.innerText || "");
+        const title = asString(await page.title());
+        const finalUrl = asString(page.url());
+        const maxAnchors = Math.max(50, Math.min(2000, Math.floor(Number(options.maxAnchors || 400))));
+        const anchors = await extractAnchors(page, maxAnchors);
+        const links = dedupeUrls(anchors.map((anchor) => asString(anchor?.href)));
+        await context.close();
+        await browser.close();
+        browser = null;
+        return {
+            ok: true,
+            method: "playwright_stealth",
+            url: options.url,
+            final_url: finalUrl || resolved.url,
+            title,
+            html,
+            text,
+            links,
+            anchors,
+            scroll_steps: driven.scroll_steps,
+            load_more_clicks: driven.load_more_clicks,
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+            ok: false,
+            method: "playwright_stealth",
+            error: {
+                category: message.toLowerCase().includes("timeout") ? "timeout" : "unavailable",
+                code: "playwright_fetch_failed",
+                message,
+            },
+        };
+    }
+    finally {
+        if (browser) {
+            await browser.close().catch(() => { });
+        }
+    }
+}
+//# sourceMappingURL=browser-session.js.map

package/dist/browser-session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-session.js","sourceRoot":"","sources":["../src/browser-session.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAEvE,MAAM,kBAAkB,GAAG,iHAAiH,CAAA;AA8C5I,IAAI,aAAa,GAAQ,SAAS,CAAA;AAElC,KAAK,UAAU,WAAW;IACxB,IAAI,aAAa,KAAK,SAAS;QAAE,OAAO,aAAa,CAAA;IACrD,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QACxD,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,gCAAgC,CAAC,CAAA;QACpE,MAAM,QAAQ,GAAI,eAAuB,CAAC,QAAQ,CAAA;QAClD,MAAM,mBAAmB,GAAI,aAAqB,CAAC,OAAO,IAAI,aAAa,CAAA;QAC3E,QAAQ,CAAC,GAAG,CAAC,mBAAmB,EAAE,CAAC,CAAA;QACnC,aAAa,GAAG,QAAQ,CAAA;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,aAAa,GAAG,KAAK,CAAA;IACvB,CAAC;IACD,OAAO,aAAa,CAAA;AACtB,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,QAAa;IACxC,MAAM,QAAQ,GAAG;QACf,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC9E,EAAE,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,QAAQ,EAAE,IAAI,EAAE;QACtE,EAAE,QAAQ,EAAE,IAAI,EAAE;KACnB,CAAA;IACD,IAAI,SAAS,GAAY,IAAI,CAAA;IAC7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,EAAE,GAAG,OAAO,EAAE,CAAA;QACpC,IAAI,aAAa,CAAC,cAAc,KAAK,SAAS;YAAE,OAAQ,aAAyC,CAAC,cAAc,CAAA;QAChH,IAAI,CAAC;YACH,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC7C,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,SAAS,GAAG,KAAK,CAAA;QACnB,CAAC;IACH,CAAC;IACD,MAAM,SAAS,CAAA;AACjB,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAS;IACpC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACxB,MAAM,OAAO,GAAG,qFAAqF,CAAA;QACrG,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,wEAAwE,CAAC,CAAC,CAAA;QAC7H,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAmB,CAAA;YACnC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;YACzI,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAQ;YAC1C,IAAI,IAAI,YAAY,iBAAiB,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;gBAC5C,IAAI,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC;oBAAE,SAAQ;YACrF,CAAC;YACD,MAAM,IAAI,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAA;YAC5C,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC;gBAAE,SAAQ;YACjD,OAAO,CAAC,KAAK,EAAE,CAAA;YACf,OAAO,IAAI,CAAA;QACb,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAS,EAAE,OAA8B;IAChE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3F,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACvG,IAAI,oBAAoB,GAAG,CAAC,CAAA;IAC5B,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,mBAAmB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;IAC9E,IAAI,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,IAAI,QAAQ,CAAC,eAAe,EAAE,YAAY,IAAI,CAAC,CAAC,CAAA;IAE1H,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,WAAW,EAAE,IAAI,EAAE,EAAE,CAAC;QAC9C,IAAI,OAAO,GAAG,KAAK,CAAA;QACnB,IAAI,cAAc,GAAG,iBAAiB,EAAE,CAAC;YACvC,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;YACtD,IAAI,OAAO,EAAE,CAAC;gBACZ,cAAc,IAAI,CAAC,CAAA;gBACnB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAA;YAChC,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,EAAE,YAAY,IAAI,QAAQ,CAAC,eAAe,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC,CAAA;QACzH,oBAAoB,IAAI,CAAC,CAAA;QACzB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAA;QAC9B,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAE7E,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,CAAA;QACjG,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,IAAI,QAAQ,CAAC,eAAe,EAAE,YAAY,IAAI,CAAC,CAAC,CAAA;QAC3H,IAAI,CAAC,OAAO,IAAI,kBAAkB,IAAI,mBAAmB,IAAI,aAAa,IAAI,cAAc,EAAE,CAAC;YAC7F,cAAc,IAAI,CAAC,CAAA;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,CAAC,CAAA;QACpB,CAAC;QACD,mBAAmB,GAAG,kBAAkB,CAAA;QACxC,cAAc,GAAG,aAAa,CAAA;QAC9B,IAAI,cAAc,IAAI,CAAC;YAAE,MAAK;IAChC,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,cAAc,EAAE,CAAA;AACjF,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAS,EAAE,UAAkB;IACzD,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAgB,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAA;QACjF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,MAAM,OAAO,GAAG,IAAyB,CAAA;YACzC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,kKAAkK,CAAuB,CAAA;YACzN,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;YACzI,MAAM,aAAa,GAAG,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,IAAI,EAAE,CAAA;YAC1F,OAAO;gBACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE;gBACxB,IAAI;gBACJ,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBAChE,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,EAAE;gBACtB,UAAU,EAAE,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;gBAC1E,UAAU,EAAE,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE;aACrD,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,EAAE,UAAU,CAAC,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAA8B;IACpE,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAA;IACpC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,oBAAoB;YAC5B,KAAK,EAAE;gBACL,QAAQ,EAAE,aAAa;gBACvB,IAAI,EAAE,wBAAwB;gBAC9B,OAAO,EAAE,2CAA2C;aACrD;SACF,CAAA;IACH,CAAC;IAED,MAAM,cAAc,GAA8C;QAChE,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAA;IACD,IAAI,OAAO,CAAC,iBAAiB,KAAK,SAAS;QAAE,cAAc,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAA;IACzG,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS;QAAE,cAAc,CAAC,OAAO,GAAG,OAAO,CAAC,YAAY,CAAA;IACrF,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,GAAG,EAAE,cAAc,CAAC,CAAA;IACvE,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,oBAAoB;YAC5B,KAAK,EAAE;gBACL,QAAQ,EAAE,OAAO;gBACjB,IAAI,EAAE,QAAQ,CAAC,MAAM;gBACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;aAC1B;SACF,CAAA;IACH,CAAC;IAED,IAAI,OAAO,GAAQ,IAAI,CAAA;IACvB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAA;QACvC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;YACvC,SAAS,EAAE,kBAAkB;YAC7B,MAAM,EAAE,OAAO;YACf,gBAAgB,EAAE,OAAO,CAAC,YAAY;SACvC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAA;QACpC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YAC5B,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,aAAa;YAC7C,OAAO,EAAE,OAAO,CAAC,SAAS;SAC3B,CAAC,CAAA;QACF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAC7C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC,CAAA;QACtE,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;QAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QACrC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QAC9F,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;QACxF,MAAM,OAAO,CAAC,KAAK,EAAE,CAAA;QACrB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAA;QACrB,OAAO,GAAG,IAAI,CAAA;QACd,OAAO;YACL,EAAE,EAAE,IAAI;YACR,MAAM,EAAE,oBAAoB;YAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,SAAS,EAAE,QAAQ,IAAI,QAAQ,CAAC,GAAG;YACnC,KAAK;YACL,IAAI;YACJ,IAAI;YACJ,KAAK;YACL,OAAO;YACP,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAA;IACH,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACtE,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,oBAAoB;YAC5B,KAAK,EAAE;gBACL,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa;gBAC/E,IAAI,EAAE,yBAAyB;gBAC/B,OAAO;aACR;SACF,CAAA;IACH,CAAC;YAAS,CAAC;QACT,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACvC,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/index.js

@@ -3,6 +3,7 @@ import { readFileSync } from "node:fs";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
+import { renderSmartcrawl, runSmartcrawl } from "./smartcrawl.js";
 import { runSmartfetch, renderSmartfetch } from "./smartfetch.js";
 import { runSmartsearch, renderSmartsearch } from "./smartsearch.js";
 const packageVersion = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
@@ -15,18 +16,72 @@ server.tool("smartsearch", "Search the web with fallback engines. Uses Exa, then
     numResults: z.number().int().min(1).max(20).optional().describe("Number of search results to return"),
     type: z.enum(["auto", "fast", "deep"]).optional().describe("Search depth hint"),
     contextMaxCharacters: z.number().int().min(1000).max(20000).optional().describe("Maximum snippet/context size hint"),
-}, async ({ query, numResults, type, contextMaxCharacters }) => {
+    site_crawl: z.boolean().optional().describe("After search, hand off into smartcrawl when a crawl_url or site: operator is available"),
+    crawl_url: z.string().optional().describe("Optional starting URL for the smartcrawl handoff"),
+    crawl_mode: z.enum(["auto", "board", "docs"]).optional().describe("Traversal hint for the smartcrawl handoff"),
+    crawl_limit: z.number().int().min(1).max(50).optional().describe("Maximum ranked crawl results to return in the handoff summary"),
+    crawl_max_pages: z.number().int().min(1).max(20).optional().describe("Maximum number of pages to visit during the crawl handoff"),
+    crawl_max_depth: z.number().int().min(0).max(3).optional().describe("Maximum recursive depth for the crawl handoff"),
+    crawl_timeout_ms: z.number().int().min(3000).max(120000).optional().describe("Per-page timeout for the crawl handoff"),
+}, async ({ query, numResults, type, contextMaxCharacters, site_crawl, crawl_url, crawl_mode, crawl_limit, crawl_max_pages, crawl_max_depth, crawl_timeout_ms }) => {
     const output = await runSmartsearch({
         query,
         ...(numResults === undefined ? {} : { numResults }),
         ...(type === undefined ? {} : { searchType: type }),
         ...(contextMaxCharacters === undefined ? {} : { contextMaxCharacters }),
+        ...(site_crawl === undefined ? {} : { siteCrawl: site_crawl }),
+        ...(crawl_url === undefined ? {} : { crawlUrl: crawl_url }),
+        ...(crawl_mode === undefined ? {} : { crawlMode: crawl_mode }),
+        ...(crawl_limit === undefined ? {} : { crawlLimit: crawl_limit }),
+        ...(crawl_max_pages === undefined ? {} : { crawlMaxPages: crawl_max_pages }),
+        ...(crawl_max_depth === undefined ? {} : { crawlMaxDepth: crawl_max_depth }),
+        ...(crawl_timeout_ms === undefined ? {} : { crawlTimeoutMs: crawl_timeout_ms }),
     });
     return {
         structuredContent: output,
         content: [{ type: "text", text: renderSmartsearch(output) }],
     };
 });
+server.tool("smartcrawl", "Explore a specific site with a browser, pagination, and scroll-aware traversal to gather relevant pages or posts.", {
+    url: z.string().min(1).describe("Starting URL to explore"),
+    query: z.string().optional().describe("Optional topic filter used to rank crawl results"),
+    mode: z.enum(["auto", "board", "docs"]).optional().describe("Traversal hint for boards, docs sites, or automatic detection"),
+    limit: z.number().int().min(1).max(50).optional().describe("Maximum number of ranked results to return"),
+    max_pages: z.number().int().min(1).max(20).optional().describe("Maximum number of pages to visit during exploration"),
+    max_depth: z.number().int().min(0).max(3).optional().describe("Maximum recursive depth from the starting URL"),
+    include_patterns: z.array(z.string().min(1)).max(20).optional().describe("Optional regex or substring patterns for URLs to keep"),
+    exclude_patterns: z.array(z.string().min(1)).max(20).optional().describe("Optional regex or substring patterns for URLs to skip"),
+    same_origin_only: z.boolean().optional().describe("Restrict traversal to the starting origin while still surfacing external item links"),
+    follow_pagination: z.boolean().optional().describe("Follow next or more-style pagination links during exploration"),
+    scroll_steps: z.number().int().min(0).max(12).optional().describe("How many scroll attempts to use for scroll-driven pages"),
+    timeout_ms: z.number().int().min(3000).max(120000).optional().describe("Crawl timeout budget per page"),
+    format: z.enum(["json", "text", "markdown"]).optional().describe("Text rendering format for the human-readable content"),
+}, async ({ url, query, mode, limit, max_pages, max_depth, include_patterns, exclude_patterns, same_origin_only, follow_pagination, scroll_steps, timeout_ms, format, }) => {
+    const output = await runSmartcrawl({
+        url,
+        ...(query === undefined ? {} : { query }),
+        ...(mode === undefined ? {} : { mode }),
+        ...(limit === undefined ? {} : { limit }),
+        ...(max_pages === undefined ? {} : { maxPages: max_pages }),
+        ...(max_depth === undefined ? {} : { maxDepth: max_depth }),
+        ...(include_patterns === undefined ? {} : { includePatterns: include_patterns }),
+        ...(exclude_patterns === undefined ? {} : { excludePatterns: exclude_patterns }),
+        ...(same_origin_only === undefined ? {} : { sameOriginOnly: same_origin_only }),
+        ...(follow_pagination === undefined ? {} : { followPagination: follow_pagination }),
+        ...(scroll_steps === undefined ? {} : { scrollSteps: scroll_steps }),
+        ...(timeout_ms === undefined ? {} : { timeoutMs: timeout_ms }),
+    });
+    const requestedFormat = format || "json";
+    return {
+        structuredContent: output,
+        content: [
+            {
+                type: "text",
+                text: requestedFormat === "json" ? JSON.stringify(output, null, 2) : renderSmartcrawl(output, requestedFormat),
+            },
+        ],
+    };
+});
 server.tool("smartfetch", "Fetch a URL with browser-aware fallbacks. Uses impit first, then stealth Playwright when the page looks JS-rendered or blocked.", {
     url: z.string().min(1).describe("Target URL to fetch"),
     target: z.enum(["auto", "generic", "reddit_post", "x_post", "dcinside_post", "youtube_video", "naver_blog", "tistory_post", "velog_post"]).optional().describe("Content type hint"),

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAA;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAA;AAChF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACjE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAEpE,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,OAAiB,CAAA;AAEtH,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;IAC3B,IAAI,EAAE,WAAW;IACjB,OAAO,EAAE,cAAc;CACxB,CAAC,CAAA;AAEF,MAAM,CAAC,IAAI,CACT,aAAa,EACb,6GAA6G,EAC7G;IACE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACrD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;IACrG,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAC/E,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;CACrH,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,oBAAoB,EAAE,EAAE,EAAE;IAC1D,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,KAAK;QACL,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACnD,GAAG,CAAC,oBAAoB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC;KACxE,CAAC,CAAA;IAEF,OAAO;QACL,iBAAiB,EAAE,MAAM;QACzB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;KAC7D,CAAA;AACH,CAAC,CACF,CAAA;AAED,MAAM,CAAC,IAAI,CACT,YAAY,EACZ,iIAAiI,EACjI;IACE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IACtD,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IACnL,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IACvG,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kDAAkD,CAAC;IAClG,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8DAA8D,CAAC;IAC9G,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,sDAAsD,CAAC;CACzH,EACD,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IAC1E,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;QACjC,GAAG;QACH,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3C,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;QAC9D,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC;QACvE,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC;KACxE,CAAC,CAAA;IAEF,MAAM,eAAe,GAAG,MAAM,IAAI,MAAM,CAAA;IACxC,OAAO;QACL,iBAAiB,EAAE,MAAM;QACzB,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,EAAE,eAAe,CAAC;aAC/G;SACF;KACF,CAAA;AACH,CAAC,CACF,CAAA;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAA;IAC5C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;AACjC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC,CAAC,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAA;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAA;AAChF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACjE,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACjE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAEpE,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,OAAiB,CAAA;AAEtH,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;IAC3B,IAAI,EAAE,WAAW;IACjB,OAAO,EAAE,cAAc;CACxB,CAAC,CAAA;AAEF,MAAM,CAAC,IAAI,CACT,aAAa,EACb,6GAA6G,EAC7G;IACE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACrD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;IACrG,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAC/E,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IACpH,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wFAAwF,CAAC;IACrI,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kDAAkD,CAAC;IAC7F,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2CAA2C,CAAC;IAC9G,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+DAA+D,CAAC;IACjI,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2DAA2D,CAAC;IACjI,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;IACpH,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC;CACvH,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,oBAAoB,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,eAAe,EAAE,gBAAgB,EAAE,EAAE,EAAE;IAC9J,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,KAAK;QACL,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACnD,GAAG,CAAC,oBAAoB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACvE,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;QAC9D,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;QAC3D,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;QAC9D,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;QACjE,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC;QAC5E,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC;QAC5E,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC;KAChF,CAAC,CAAA;IAEF,OAAO;QACL,iBAAiB,EAAE,MAAM;QACzB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;KAC7D,CAAA;AACH,CAAC,CACF,CAAA;AAED,MAAM,CAAC,IAAI,CACT,YAAY,EACZ,mHAAmH,EACnH;IACE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC1D,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kDAAkD,CAAC;IACzF,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+DAA+D,CAAC;IAC5H,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4CAA4C,CAAC;IACxG,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qDAAqD,CAAC;IACrH,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;IAC9G,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uDAAuD,CAAC;IACjI,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uDAAuD,CAAC;IACjI,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qFAAqF,CAAC;IACxI,iBAAiB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+DAA+D,CAAC;IACnH,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yDAAyD,CAAC;IAC5H,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IACvG,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,sDAAsD,CAAC;CACzH,EACD,KAAK,EAAE,EACL,GAAG,EACH,KAAK,EACL,IAAI,EACJ,KAAK,EACL,SAAS,EACT,SAAS,EACT,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,MAAM,GACP,EAAE,EAAE;IACH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;QACjC,GAAG;QACH,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;QACzC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACvC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;QACzC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;QAC3D,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;QAC3D,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,gBAAgB,EAAE,CAAC;QAChF,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,gBAAgB,EAAE,CAAC;QAChF,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC;QAC/E,GAAG,CAAC,iBAAiB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,CAAC;QACnF,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC;QACpE,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;KAC/D,CAAC,CAAA;IAEF,MAAM,eAAe,GAAG,MAAM,IAAI,MAAM,CAAA;IACxC,OAAO;QACL,iBAAiB,EAAE,MAAM;QACzB,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,EAAE,eAAe,CAAC;aAC/G;SACF;KACF,CAAA;AACH,CAAC,CACF,CAAA;AAED,MAAM,CAAC,IAAI,CACT,YAAY,EACZ,iIAAiI,EACjI;IACE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IACtD,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IACnL,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IACvG,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kDAAkD,CAAC;IAClG,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8DAA8D,CAAC;IAC9G,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,sDAAsD,CAAC;CACzH,EACD,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IAC1E,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;QACjC,GAAG;QACH,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3C,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;QAC9D,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC;QACvE,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC;KACxE,CAAC,CAAA;IAEF,MAAM,eAAe,GAAG,MAAM,IAAI,MAAM,CAAA;IACxC,OAAO;QACL,iBAAiB,EAAE,MAAM;QACzB,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,EAAE,eAAe,CAAC;aAC/G;SACF;KACF,CAAA;AACH,CAAC,CACF,CAAA;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAA;IAC5C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;AACjC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC,CAAC,CAAA"}

package/dist/shared.d.ts

@@ -19,12 +19,33 @@ export type SearchResult = {
     snippet: string;
     engine: string;
 };
+export type SearchHandoffItem = {
+    title: string;
+    url: string;
+    snippet: string;
+    kind: string;
+    score: number;
+    depth: number;
+};
+export type SearchHandoff = {
+    tool: "smartcrawl";
+    performed: boolean;
+    reason: string;
+    start_url: string;
+    query: string;
+    mode: "auto" | "board" | "docs";
+    items: SearchHandoffItem[];
+    pages_attempted: number;
+    pages_loaded: number;
+    errors: string[];
+};
 export type SearchResponse = {
     engine: string;
     query: string;
     results: SearchResult[];
     fallbacks_tried: string[];
     notes: string[];
+    handoff?: SearchHandoff;
 };
 export type SmartfetchOutput = {
     source: "smartfetch";
@@ -37,6 +58,7 @@ export type SmartfetchOutput = {
     post: Record<string, unknown> | null;
     thread: Record<string, unknown>[];
     comments: Record<string, unknown>[];
+    assets: SmartfetchAsset[];
     outbound_links: string[];
     evidence: {
         impit_attempted: boolean;
@@ -46,7 +68,18 @@ export type SmartfetchOutput = {
         generated_at: string;
     };
 };
+export type SmartfetchAssetKind = "image" | "video" | "audio" | "document" | "archive" | "attachment" | "other";
+export type SmartfetchAsset = {
+    url: string;
+    download_url: string;
+    kind: SmartfetchAssetKind;
+    file_name: string;
+    mime_type: string;
+    source: string;
+    downloadable: boolean;
+};
 export type SmartfetchRenderFormat = "text" | "markdown";
+export type SmartWebProfile = "balanced" | "private";
 export declare function asString(value: unknown): string;
 export declare function asNumber(value: unknown): number;
 export declare function decodeHtml(value: string): string;
@@ -56,6 +89,12 @@ export declare function dedupeUrls(values: string[]): string[];
 export declare function cleanLinks(values: string[]): string[];
 export declare function extractUrls(value: string): string[];
 export declare function extractAnchorHrefs(value: string): string[];
+export declare function absolutizeUrl(input: string, baseUrl: string): string;
+export declare function extractTagAttribute(html: string, tag: string, attribute: string, baseUrl: string): string[];
+export declare function extractMetaContentValues(html: string, names: string[], baseUrl: string): string[];
+export declare function fileNameFromUrl(url: string): string;
+export declare function inferMimeType(url: string): string;
+export declare function inferAssetKind(url: string, mimeType?: string): SmartfetchAssetKind;
 export declare function uniqueResults(results: SearchResult[]): SearchResult[];
 export declare function resolveDuckDuckGoUrl(input: string): string;
 export declare function capture(value: string, regex: RegExp): string;
@@ -64,8 +103,9 @@ export declare function extractMetaDescription(html: string): string;
 export declare function extractMetaProperty(html: string, property: string): string;
 export declare function extractMetaName(html: string, name: string): string;
 export declare function inferTarget(url: string, target: Target): ResolvedTarget;
+export declare function smartWebProfile(): SmartWebProfile;
 export declare function envFlag(name: string): boolean;
-export declare function envEnabled(name: string, defaultValue: boolean): boolean;
+export declare function envEnabled(name: string, defaultValue: boolean, profileDefaults?: Partial<Record<SmartWebProfile, boolean>>): boolean;
 export declare function validateOutboundUrl(url: string, options?: {
     allowPrivateHosts?: boolean;
 }): Promise<{
@@ -77,6 +117,19 @@ export declare function validateOutboundUrl(url: string, options?: {
     reason?: never;
     message?: never;
 }>;
+export declare function resolveValidatedUrl(url: string, options?: {
+    allowPrivateHosts?: boolean;
+    timeoutMs?: number;
+    maxRedirects?: number;
+    headers?: HeadersInit;
+}): Promise<{
+    ok: false;
+    reason: string;
+    message: string;
+} | {
+    ok: true;
+    url: string;
+}>;
 export declare function isXUrl(url: string): boolean;
 export declare function isRedditUrl(url: string): boolean;
 export declare function isDcinsideUrl(url: string): boolean;