searchfetch 3.0.0 → 3.1.0

package/index.js

@@ -244,6 +244,33 @@ function mapSearchParams(engine, query, region, safeSearch) {
 
 // === FETCH ===============================================================
 
+const FETCH_MAX_ATTEMPTS = 2;
+const HTTP_429_RETRY_DELAY_MS = 2000;
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function parseRetryAfterMs(value) {
+  if (!value) return HTTP_429_RETRY_DELAY_MS;
+  const seconds = Number(value);
+  if (Number.isFinite(seconds) && seconds >= 0) {
+    return Math.min(seconds * 1000, 30000);
+  }
+  const dateMs = Date.parse(value);
+  if (Number.isFinite(dateMs)) {
+    return Math.min(Math.max(dateMs - Date.now(), 0), 30000);
+  }
+  return HTTP_429_RETRY_DELAY_MS;
+}
+
+function makeHttpStatusError(status, url, retryAfterMs = null) {
+  const err = new Error(`Access denied: HTTP ${status} when fetching ${url}`);
+  err.httpStatus = status;
+  err.retryAfterMs = retryAfterMs;
+  return err;
+}
+
 function isAccessDenied($) {
   const title = ($("title").text() || "").toLowerCase();
   const bodyText = ($("body").text() || "").replace(/\s+/g, " ").trim().toLowerCase();
@@ -325,8 +352,10 @@ async function fetchHtml(url, template, blockMedia) {
       if (response) {
         const status = response.status();
         if ([401, 403, 429].includes(status)) {
-          throw new Error(
-            `Access denied: HTTP ${status} when fetching ${url}`,
+          throw makeHttpStatusError(
+            status,
+            url,
+            status === 429 ? parseRetryAfterMs(response.headers()["retry-after"]) : null,
           );
         }
       }
@@ -352,18 +381,25 @@ async function fetchHtml(url, template, blockMedia) {
 
 async function fetchHtmlWithRetry(url, template, blockMedia) {
   let lastError;
-  for (let attempt = 0; attempt < 2; attempt++) {
+  for (let attempt = 0; attempt < FETCH_MAX_ATTEMPTS; attempt++) {
     try {
       return await fetchHtml(url, template, blockMedia);
     } catch (err) {
       lastError = err;
       if (
-        attempt === 0 &&
+        attempt < FETCH_MAX_ATTEMPTS - 1 &&
+        err.httpStatus === 429
+      ) {
+        await sleep(err.retryAfterMs ?? HTTP_429_RETRY_DELAY_MS);
+        continue;
+      }
+      if (
+        attempt < FETCH_MAX_ATTEMPTS - 1 &&
         (err.message.includes("net::") ||
           err.message.includes("ERR_") ||
           err.message.includes("Navigation failed"))
       ) {
-        // Network error — retry once
+        await sleep(500);
         continue;
       }
       throw err;
@@ -372,6 +408,100 @@ async function fetchHtmlWithRetry(url, template, blockMedia) {
   throw lastError;
 }
 
+// === SOURCE MARKDOWN DETECTION ===========================================
+
+function isMarkdownContent(text) {
+  if (!text) return false;
+  const htmlTagCount = (text.match(/<\w+[^>]*>/g) || []).length;
+  if (htmlTagCount > 3) return false;
+  const patterns = [
+    /^#{1,6}\s+\S/m,
+    /\[.+?\]\(.+?\)/,
+    /```\w*\n/,
+    /^\s*[-*+]\s+\S/m,
+    /\*\*[^*]+\*\*/,
+    /^>\s+\S/m,
+  ];
+  for (const pat of patterns) {
+    if (pat.test(text)) return true;
+  }
+  return false;
+}
+
+function stripSourceMarkdown(content) {
+  return content
+    .replace(/^@twoslash-cache:.*$/gm, "")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+function resolveSourceUrl(sourceTemplate, url) {
+  if (sourceTemplate === "{url}.md") {
+    return `${url.replace(/\/+$/, "")}.md`;
+  }
+  return sourceTemplate.replace("{url}", url);
+}
+
+async function fetchSourceMarkdown(sourceUrl, template, blockMedia) {
+  const browser = await browserManager.getBrowser();
+  const context = await browser.newContext();
+
+  try {
+    if (template && template.cookies && template.cookies.length > 0) {
+      await context.addCookies(template.cookies);
+    }
+
+    const page = await context.newPage();
+    try {
+      if (blockMedia) {
+        const blockedTypes =
+          template && template.block_resources
+            ? template.block_resources
+            : ["image", "media", "font"];
+        if (blockedTypes.length > 0) {
+          await page.route("**/*", (route) => {
+            const type = route.request().resourceType();
+            if (blockedTypes.includes(type)) route.abort();
+            else route.continue();
+          });
+        }
+      }
+
+      let response;
+      try {
+        response = await page.goto(sourceUrl, {
+          waitUntil: "domcontentloaded",
+          timeout: 10000,
+        });
+      } catch (_) {
+        return null;
+      }
+
+      if (response && response.status() >= 400) return null;
+
+      let text;
+      try {
+        text = await page.evaluate(
+          "() => document.body?.innerText || document.body?.textContent || ''",
+        );
+      } catch (_) {
+        text = await page.content();
+      }
+
+      if (!text || typeof text !== "string") return null;
+
+      text = stripSourceMarkdown(text.trim());
+      return isMarkdownContent(text) ? text : null;
+    } finally {
+      await page.close();
+    }
+  } catch (_) {
+    return null;
+  } finally {
+    await context.close();
+  }
+}
+
 // === HTML CLEANUP ========================================================
 
 const DEFAULT_REMOVE_SELECTORS = [
@@ -920,7 +1050,7 @@ function resolveSearchTemplate(engine, query, region, safeSearch) {
 
 // === MCP SERVER & TOOLS ==================================================
 
-const server = new McpServer({ name: "searchfetch", version: "3.0.0" });
+const server = new McpServer({ name: "searchfetch", version: "3.0.1" });
 
 // --- websearch tool ---
 
@@ -1059,10 +1189,31 @@ server.registerTool(
         template = getTemplateByName(templateParam);
       }
 
-      // 2. Fetch
+      // 2. Try source markdown if template specifies source_url
+      let sourceMd = null;
+      if (template && template.source_url) {
+        const sourceUrl = resolveSourceUrl(template.source_url, url);
+        sourceMd = await fetchSourceMarkdown(sourceUrl, template, block_media);
+      }
+
+      if (sourceMd !== null) {
+        const totalLength = sourceMd.length;
+        const paginated = sourceMd.substring(start_index, start_index + max_length);
+        let metadata =
+          `\n\n---\n[webfetch: template="${template ? template.name : "auto"}" (source markdown), ` +
+          `showing characters ${start_index} to ${start_index + paginated.length} of ${totalLength} total.`;
+        if (start_index + max_length < totalLength) {
+          metadata +=
+            ` Use start_index=${start_index + max_length} to read more.`;
+        }
+        metadata += "]";
+        return { content: [{ type: "text", text: paginated + metadata }] };
+      }
+
+      // 3. Fetch
       const html = await fetchHtmlWithRetry(url, template, block_media);
 
-      // 3. Extract and compose
+      // 4. Extract and compose
       const $ = cheerio.load(html);
 
       if (template) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "searchfetch",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "Fault-tolerant MCP Server for Stealth Web Search and Fetching",
   "type": "module",
   "bin": {

package/templates/docker-hub.json

@@ -0,0 +1,41 @@
+{
+  "name": "docker-hub",
+  "description": "Docker Hub image page — official and community images with description and README",
+  "order": 10,
+  "url_patterns": [
+    "^https?://hub\\.docker\\.com/_/[^/]+/?$",
+    "^https?://hub\\.docker\\.com/r/[^/]+/[^/]+/?$"
+  ],
+  "remove": [
+    "script",
+    "style",
+    "svg",
+    "nav",
+    "footer",
+    "header",
+    ".styles_sidebar__",
+    ".styles_chatbot__",
+    "[data-testid='chatbot']"
+  ],
+  "sections": [
+    {
+      "name": "Image",
+      "selector": "h1",
+      "format": "text",
+      "required": false
+    },
+    {
+      "name": "Description",
+      "selector": "meta[name='description']",
+      "format": "attribute",
+      "attribute": "content",
+      "required": false
+    },
+    {
+      "name": "Content",
+      "selector": "#readme-more-content-wrapper, article, [data-testid='description'], main, .MuiContainer-root",
+      "format": "markdown",
+      "required": false
+    }
+  ]
+}

package/templates/docs-page.json

@@ -1,29 +1,40 @@
 {
   "name": "docs-page",
-  "description": "Documentation page (ReadTheDocs, Sphinx, etc.)",
-  "order": 8,
+  "description": "Documentation page (ReadTheDocs, Sphinx, Mintlify, VitePress, Docusaurus, Nextra, etc.)",
+  "order": 9,
   "url_patterns": [
     "^https?://[^/]+\\.readthedocs\\.io/.*",
-    "^https?://[^/]+\\.rtfd\\.io/.*"
+    "^https?://[^/]+\\.rtfd\\.io/.*",
+    "^https?://docs\\.mintlify\\.com/.*",
+    "^https?://[^/]+\\.mintlify\\.(?:com|app|dev)/.*",
+    "^https?://[^/]+/docs(?:/.*)?$",
+    "^https?://[^/]+/doc(?:/.*)?$",
+    "^https?://docs\\.[^/]+\\.[^/]+/.*"
   ],
+  "source_url": "{url}.md",
   "remove": [
     "script",
     "style",
     "svg",
     "nav",
     "footer",
-    ".sphinxsidebar"
+    ".sphinxsidebar",
+    ".sidebar",
+    "aside",
+    ".nextra-toc",
+    ".table-of-contents",
+    ".pagination-nav"
   ],
   "sections": [
     {
       "name": "Title",
-      "selector": "h1",
+      "selector": "h1, h2",
       "format": "text",
       "required": false
     },
     {
       "name": "Content",
-      "selector": "[role='main'], .document, .rst-content, article",
+      "selector": "[role='main'], .document, .rst-content, article, .content, .markdown-body, .prose, main, .nextra-content, .doc-content",
       "format": "markdown",
       "required": false
     }

package/templates/docs-rs.json

@@ -0,0 +1,47 @@
+{
+  "name": "docs-rs",
+  "description": "docs.rs rustdoc documentation page — crate docs, struct/enum/fn items",
+  "order": 6,
+  "url_patterns": [
+    "^https?://docs\\.rs/[^/]+/[^/]+/[^/]+/?$",
+    "^https?://docs\\.rs/[^/]+/[^/]+/[^/]+/index\\.html$",
+    "^https?://docs\\.rs/[^/]+/[^/]+/[^/]+/.+\\.html$"
+  ],
+  "remove": [
+    "script",
+    "style",
+    "svg",
+    "nav.sidebar",
+    "nav.sub",
+    "footer",
+    ".search-form",
+    ".rustdoc-version-dialog",
+    ".mobile-topbar",
+    ".out-of-band",
+    "#source-sidebar",
+    "#theme-picker",
+    "#settings-menu",
+    ".since",
+    ".srclink"
+  ],
+  "sections": [
+    {
+      "name": "Crate",
+      "selector": "h1.crate-title, .crate-title h1, .in-band .crate-name",
+      "format": "text",
+      "required": false
+    },
+    {
+      "name": "Title",
+      "selector": "h1.fqn, .in-band h1, h1",
+      "format": "text",
+      "required": false
+    },
+    {
+      "name": "Content",
+      "selector": "#main-content, main, .main, .docblock",
+      "format": "markdown",
+      "required": false
+    }
+  ]
+}

package/templates/duckduckgo-search.json

@@ -1,7 +1,7 @@
 {
   "name": "duckduckgo-search",
   "description": "DuckDuckGo HTML search results",
-  "order": 7,
+  "order": 8,
   "url_patterns": [
     "^https?://html\\.duckduckgo\\.com/html/\\?"
   ],

package/templates/google-search.json

@@ -1,7 +1,7 @@
 {
   "name": "google-search",
   "description": "Google web search results",
-  "order": 6,
+  "order": 7,
   "url_patterns": [
     "^https?://(www\\.)?google\\.[^/]+/search\\?"
   ],