searchfetch 2.0.0 → 3.0.0

package/index.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 
+// === STDOUT/STDERR REDIRECTION ===========================================
 const originalStdoutWrite = process.stdout.write.bind(process.stdout);
 process.stdout.write = (chunk, encoding, callback) => {
   return process.stderr.write(chunk, encoding, callback);
@@ -7,6 +8,10 @@ process.stdout.write = (chunk, encoding, callback) => {
 console.log = (...args) => console.error(...args);
 console.info = (...args) => console.error(...args);
 
+// === IMPORTS =============================================================
+import { readdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
@@ -14,6 +19,7 @@ import { launch, ensureBinary } from "cloakbrowser";
 import * as cheerio from "cheerio";
 import TurndownService from "turndown";
 
+// === BROWSER MANAGER =====================================================
 class BrowserManager {
   constructor() {
     this.browser = null;
@@ -67,225 +73,856 @@ const cleanup = async () => {
 process.on("SIGINT", cleanup);
 process.on("SIGTERM", cleanup);
 
-function getGoogleRegionParams(region) {
-  if (!region || region === "wt-wt") return "hl=en&gl=us";
-  const parts = region.split("-");
-  if (parts.length === 2) return `gl=${parts[0]}&hl=${parts[1]}`;
-  return `gl=${region}&hl=en`;
+// === TURNDOWN ============================================================
+const turndown = new TurndownService({
+  headingStyle: "atx",
+  codeBlockStyle: "fenced",
+  emDelimiter: "*",
+});
+
+// === BUILT-IN TEMPLATES (loaded from templates/*.json) ====================
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const TEMPLATES_DIR = join(__dirname, "templates");
+
+function loadBuiltinTemplates() {
+  let files;
+  try {
+    files = readdirSync(TEMPLATES_DIR);
+  } catch (err) {
+    throw new Error(
+      `Cannot read templates directory '${TEMPLATES_DIR}': ${err.message}`,
+    );
+  }
+
+  const jsonFiles = files
+    .filter((f) => f.endsWith(".json"))
+    .sort();
+
+  if (jsonFiles.length === 0) {
+    throw new Error(
+      `No template JSON files found in '${TEMPLATES_DIR}'`,
+    );
+  }
+
+  const templates = [];
+  for (const file of jsonFiles) {
+    const filePath = join(TEMPLATES_DIR, file);
+    const content = readFileSync(filePath, "utf-8");
+    let template;
+    try {
+      template = JSON.parse(content);
+    } catch (err) {
+      throw new Error(
+        `Invalid JSON in template file '${filePath}': ${err.message}`,
+      );
+    }
+    if (!template.name || typeof template.name !== "string") {
+      throw new Error(
+        `Template file '${filePath}' is missing a valid "name" field`,
+      );
+    }
+    templates.push(template);
+  }
+
+  // Sort by "order" field for deterministic URL-pattern matching
+  templates.sort((a, b) => (a.order ?? 999) - (b.order ?? 999));
+
+  return templates;
+}
+
+const BUILTIN_TEMPLATES = loadBuiltinTemplates();
+
+// === TEMPLATE LOOKUP =====================================================
+const TEMPLATE_MAP = new Map();
+for (const t of BUILTIN_TEMPLATES) {
+  TEMPLATE_MAP.set(t.name, t);
+}
+
+function getTemplateByName(name) {
+  const t = TEMPLATE_MAP.get(name);
+  if (!t) {
+    const names = [...TEMPLATE_MAP.keys()].join(", ");
+    throw new Error(
+      `Unknown template '${name}'. Available: ${names}`,
+    );
+  }
+  return t;
+}
+
+function detectTemplateByUrl(url) {
+  for (const template of BUILTIN_TEMPLATES) {
+    if (!template.url_patterns) continue;
+    for (const pattern of template.url_patterns) {
+      try {
+        if (new RegExp(pattern).test(url)) {
+          return template;
+        }
+      } catch (_) {
+        // Skip invalid regex
+      }
+    }
+  }
+  return null;
+}
+
+// === URL TEMPLATE RESOLUTION =============================================
+
+function resolveUrlTemplate(template, providedParams) {
+  const urlParams = template.url_params || {};
+  let url = template.url_template;
+  if (!url) return null;
+
+  let match;
+  const re = /\{(\w+)\}/g;
+  while ((match = re.exec(url)) !== null) {
+    const name = match[1];
+    const def = urlParams[name] || {};
+
+    let value;
+    if (
+      name in providedParams &&
+      providedParams[name] !== null &&
+      providedParams[name] !== undefined
+    ) {
+      value = String(providedParams[name]);
+    } else if (def.default !== undefined) {
+      value = String(def.default);
+    } else if (def.required) {
+      throw new Error(
+        `Required URL parameter '${name}' not provided for template '${template.name}'.`,
+      );
+    } else {
+      value = "";
+    }
+
+    if (def.encode === "url") {
+      value = encodeURIComponent(value);
+    }
+
+    url = url.replace(match[0], value);
+  }
+
+  // Remove any remaining unreplaced placeholders
+  url = url.replace(/\{\w+\}/g, "").replace(/&{2,}/g, "&").replace(/\?&/, "?");
+
+  return url;
+}
+
+// === SEARCH PARAM MAPPING ================================================
+
+function resolveEngineToTemplateName(engine) {
+  if (engine === "duckduckgo") return "duckduckgo-search";
+  if (engine === "google") return "google-search";
+  return engine;
+}
+
+function mapSearchParams(engine, query, region, safeSearch) {
+  const params = { query };
+  const resolved = resolveEngineToTemplateName(engine);
+
+  if (resolved === "duckduckgo-search") {
+    if (region !== null && region !== undefined) {
+      params.kl = region;
+    }
+    if (safeSearch === true) {
+      params.kp = "1";
+    } else if (safeSearch === false) {
+      params.kp = "-2";
+    }
+  } else if (resolved === "google-search") {
+    if (region !== null && region !== undefined) {
+      const parts = region.split("-");
+      params.hl = parts[0];
+      params.gl = parts.length > 1 ? parts[1] : parts[0];
+    }
+  }
+
+  return params;
+}
+
+// === FETCH ===============================================================
+
+function isAccessDenied($) {
+  const title = ($("title").text() || "").toLowerCase();
+  const bodyText = ($("body").text() || "").replace(/\s+/g, " ").trim().toLowerCase();
+
+  const titleDenyPatterns = [
+    "captcha",
+    "are you a robot",
+    "access denied",
+    "blocked",
+    "forbidden",
+    "unusual traffic",
+    "sorry, you have been blocked",
+    "verify you are human",
+    "one more step",
+    "security check",
+    "ddos protection",
+    "cloudflare",
+  ];
+
+  if (titleDenyPatterns.some((pattern) => title.includes(pattern))) return true;
+
+  const bodyDenyPatterns = [
+    "to continue, please type the characters",
+    "our systems have detected unusual traffic",
+    "verify you are human",
+    "are you a robot",
+    "sorry, you have been blocked",
+    "access denied",
+  ];
+
+  if (bodyText.length < 1200 && bodyDenyPatterns.some((pattern) => bodyText.includes(pattern))) return true;
+
+  return false;
 }
 
-async function executeSearch(query, maxResults, region, safeSearch, engine) {
+async function fetchHtml(url, template, blockMedia) {
   const browser = await browserManager.getBrowser();
   const context = await browser.newContext();
 
-  await context.addCookies([
-    {
-      name: "CONSENT",
-      value: "YES+cb.20250101-01-p0.en+FX+999",
-      domain: ".google.com",
-      path: "/",
-    },
-  ]);
+  try {
+    // Pre-load cookies from template
+    if (template && template.cookies && template.cookies.length > 0) {
+      await context.addCookies(template.cookies);
+    }
 
-  const page = await context.newPage();
+    const page = await context.newPage();
 
-  try {
-    await page.route("**/*", (route) => {
-      const type = route.request().resourceType();
-      if (["image", "media", "font", "stylesheet"].includes(type)) {
-        route.abort();
-      } else {
-        route.continue();
+    try {
+      // Route blocked resource types
+      if (blockMedia) {
+        const blockedTypes =
+          template && template.block_resources
+            ? template.block_resources
+            : ["image", "media", "font"];
+
+        if (blockedTypes.length > 0) {
+          await page.route("**/*", (route) => {
+            const type = route.request().resourceType();
+            if (blockedTypes.includes(type)) {
+              route.abort();
+            } else {
+              route.continue();
+            }
+          });
+        }
       }
-    });
 
-    const results = [];
-    const searchUrl =
-      engine === "google"
-        ? `https://www.google.com/search?udm=web&udm=14&q=${encodeURIComponent(query)}&${getGoogleRegionParams(region)}`
-        : `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}&kl=${encodeURIComponent(region)}&kp=${encodeURIComponent(safeSearch)}`;
+      let response;
+      try {
+        response = await page.goto(url, {
+          waitUntil: "networkidle",
+          timeout: 15000,
+        });
+      } catch (_navError) {
+        // Allow partial rendering on timeout
+      }
 
+      // Check HTTP status for access failures
+      if (response) {
+        const status = response.status();
+        if ([401, 403, 429].includes(status)) {
+          throw new Error(
+            `Access denied: HTTP ${status} when fetching ${url}`,
+          );
+        }
+      }
+
+      const pageContent = await page.content();
+
+      // Check for CAPTCHA / access-denied pages
+      const $ = cheerio.load(pageContent);
+      if (isAccessDenied($)) {
+        throw new Error(
+          `Access denied: CAPTCHA or block page detected at ${url}. The site is blocking automated access.`,
+        );
+      }
+
+      return pageContent;
+    } finally {
+      await page.close();
+    }
+  } finally {
+    await context.close();
+  }
+}
+
+async function fetchHtmlWithRetry(url, template, blockMedia) {
+  let lastError;
+  for (let attempt = 0; attempt < 2; attempt++) {
     try {
-      await page.goto(searchUrl, { waitUntil: "networkidle", timeout: 15000 });
-    } catch (e) {
-      // Allow partial rendering on timeout
+      return await fetchHtml(url, template, blockMedia);
+    } catch (err) {
+      lastError = err;
+      if (
+        attempt === 0 &&
+        (err.message.includes("net::") ||
+          err.message.includes("ERR_") ||
+          err.message.includes("Navigation failed"))
+      ) {
+        // Network error — retry once
+        continue;
+      }
+      throw err;
+    }
+  }
+  throw lastError;
+}
+
+// === HTML CLEANUP ========================================================
+
+const DEFAULT_REMOVE_SELECTORS = [
+  "script", "style", "svg", "nav", "footer", "noscript", "iframe",
+  ".advertisement",
+];
+
+function applyRemove($, template) {
+  const removeSelectors =
+    template && template.remove && template.remove.length > 0
+      ? template.remove
+      : DEFAULT_REMOVE_SELECTORS;
+
+  for (const selector of removeSelectors) {
+    try {
+      $(selector).remove();
+    } catch (_) {
+      // Skip invalid selectors
     }
+  }
 
-    const pageContent = await page.content();
-    const $ = cheerio.load(pageContent);
+  // Strip style attributes and data:image src
+  $("[style]").removeAttr("style");
+  $("*").each((_i, el) => {
+    const src = $(el).attr("src");
+    if (src && src.startsWith("data:image")) {
+      $(el).removeAttr("src");
+    }
+  });
+}
 
-    if (engine === "google") {
-      $("h3").each((i, el) => {
-        if (results.length >= maxResults) return;
+// === EXTRACTION ENGINE ===================================================
 
-        const h3 = $(el);
-        let linkEl = h3.closest("a");
-        if (!linkEl.length) linkEl = h3.find("a");
-        if (!linkEl.length) return;
+/**
+ * Find elements matching selector, scoped to $parent.
+ * Search order: descendants → closest ancestor → ancestor subtrees (up to 4 levels).
+ */
+function findScoped($parent, selector) {
+  if (!selector || selector.trim() === "") {
+    return $parent;
+  }
 
-        let link = linkEl.attr("href") || "";
+  // 1. Descendants
+  let result = $parent.find(selector);
+  if (result.length > 0) return result;
 
-        if (!link || (link.startsWith("/") && !link.startsWith("/url?q=")))
-          return;
-        if (
-          link.includes("google.com/search") ||
-          link.includes("support.google.com")
-        )
-          return;
+  // 2. Closest ancestor matching selector
+  result = $parent.closest(selector);
+  if (result.length > 0) return result;
 
-        if (link.startsWith("/url?q=")) {
-          try {
-            link = decodeURIComponent(link.split("/url?q=")[1].split("&")[0]);
-          } catch (e) {}
-        }
+  // 3. Ancestor subtrees (up to 4 levels up)
+  let ancestor = $parent.parent();
+  for (let i = 0; i < 4 && ancestor.length > 0; i++) {
+    result = ancestor.find(selector);
+    if (result.length > 0) return result;
+    ancestor = ancestor.parent();
+  }
 
-        const title = h3.text().trim();
-        if (!title) return;
+  return $parent.find("__nonexistent__");
+}
 
-        let snippet = "";
-        let parent = h3.parent();
-        while (parent.length && parent.prop("tagName") !== "BODY") {
-          const snippetEl = parent.find(
-            "div.VwiC3b, div[style*='-webkit-line-clamp'], div.yXK7lf, div.Uroaid",
-          );
-          if (snippetEl.length) {
-            snippet = snippetEl.first().text().replace(/\s+/g, " ").trim();
-            break;
+/**
+ * Try comma-separated selectors in order; first match wins.
+ */
+function findFirstMatch($parent, selectorStr) {
+  if (!selectorStr || selectorStr.trim() === "") {
+    return $parent;
+  }
+
+  const selectors = selectorStr
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
+
+  for (const sel of selectors) {
+    const matches = findScoped($parent, sel);
+    if (matches.length > 0) return matches;
+  }
+
+  return $parent.find("__nonexistent__");
+}
+
+/**
+ * Resolve top-level elements for a section (document-wide with fallback).
+ */
+function resolveTopElements($, selectorStr) {
+  if (!selectorStr || selectorStr.trim() === "") {
+    return $("body");
+  }
+
+  const selectors = selectorStr
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
+
+  for (const sel of selectors) {
+    try {
+      const matches = $(sel);
+      if (matches.length > 0) return matches;
+    } catch (_) {
+      // Skip invalid selectors
+    }
+  }
+
+  return $();
+}
+
+// === TRANSFORMS ==========================================================
+
+function applyTransform(value, transform, origin) {
+  const transforms = Array.isArray(transform) ? transform : [transform];
+  let result = value;
+
+  for (const t of transforms) {
+    if (!result) continue;
+    switch (t) {
+      case "strip":
+        result = result.trim();
+        break;
+
+      case "decode_google_url":
+        if (result.startsWith("/url?q=")) {
+          try {
+            const urlPart = result.split("/url?q=")[1].split("&")[0];
+            result = decodeURIComponent(urlPart);
+          } catch (_) {
+            // Leave as-is
           }
-          parent = parent.parent();
         }
+        break;
 
-        if (link.startsWith("http")) {
-          if (!results.some((r) => r.link === link)) {
-            results.push({
-              position: results.length + 1,
-              title,
-              link,
-              snippet,
-            });
+      case "decode_ddg_url":
+        if (result.includes("/l/?uddg=")) {
+          try {
+            const queryString = result.split("?")[1] || "";
+            const params = new URLSearchParams(queryString);
+            const uddg = params.get("uddg");
+            if (uddg) result = decodeURIComponent(uddg);
+          } catch (_) {
+            // Leave as-is
           }
         }
-      });
-    } else {
-      $(".result").each((i, el) => {
-        if (results.length >= maxResults) return;
-        const titleEl = $(el).find(".result__title a");
-        let link = titleEl.attr("href") || "";
+        break;
 
-        if (link.includes("/l/?uddg=")) {
+      case "json_parse":
+        try {
+          result = JSON.stringify(JSON.parse(result), null, 2);
+        } catch (_) {
+          // Leave as-is
+        }
+        break;
+
+      case "resolve_href":
+        if (origin && result.startsWith("/") && !result.startsWith("//")) {
           try {
-            const urlParams = new URLSearchParams(link.split("?")[1]);
-            link = decodeURIComponent(urlParams.get("uddg") || link);
-          } catch (e) {}
+            result = new URL(result, origin).href;
+          } catch (_) {
+            // Leave as-is
+          }
         }
+        break;
+    }
+  }
+
+  return result;
+}
+
+// === EXTRACTION ==========================================================
+
+function extractValue($el, section, origin) {
+  let value;
+
+  switch (section.format) {
+    case "text":
+      value = $el.text().replace(/\s+/g, " ").trim();
+      break;
+
+    case "markdown": {
+      const html = $el.html() || "";
+      value = turndown
+        .turndown(html)
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+      break;
+    }
+
+    case "attribute":
+      value = $el.attr(section.attribute) || "";
+      break;
+
+    case "html":
+      value = $el.html() || "";
+      break;
+
+    default:
+      value = $el.text().replace(/\s+/g, " ").trim();
+  }
+
+  if (section.transform && value) {
+    value = applyTransform(value, section.transform, origin);
+  }
 
-        const title = titleEl.text().trim();
-        const snippet = $(el)
-          .find(".result__snippet")
-          .text()
-          .replace(/\s+/g, " ")
-          .trim();
+  return value;
+}
+
+/**
+ * Extract one child section, scoped to $parentEl.
+ * Returns { type: "value", text } or null.
+ */
+function extractChildSection($, $parentEl, section, origin) {
+  const elements = findFirstMatch($parentEl, section.selector);
+
+  if (!elements || elements.length === 0) {
+    if (section.required) {
+      throw new Error(
+        `Required section '${section.name}' not found on page.`,
+      );
+    }
+    return null;
+  }
+
+  const el = elements.eq(0);
+  const value = extractValue(el, section, origin);
+  return { type: "value", text: value };
+}
 
-        if (title && link.startsWith("http")) {
-          results.push({ position: results.length + 1, title, link, snippet });
+/**
+ * Extract a top-level section from the document.
+ * Returns a SectionResult or null.
+ */
+function extractSection($, section, context) {
+  const elements = resolveTopElements($, section.selector);
+
+  if (!elements || elements.length === 0) {
+    if (section.required) {
+      throw new Error(
+        `Required section '${section.name}' not found on page.`,
+      );
+    }
+    return null;
+  }
+
+  // Determine limit
+  let limit = elements.length;
+  if (section.multiple && section.max_items) {
+    limit = Math.min(limit, section.max_items);
+  }
+  // Override max_items with max_results for first multiple+children section
+  if (
+    context.isWebsearch &&
+    context.maxResultsOverride &&
+    !context._maxResultsConsumed &&
+    section.multiple &&
+    section.children &&
+    section.children.length > 0
+  ) {
+    limit = Math.min(limit, context.maxResultsOverride);
+    context._maxResultsConsumed = true;
+  }
+
+  if (section.multiple) {
+    const items = [];
+
+    for (let i = 0; i < limit; i++) {
+      const el = elements.eq(i);
+
+      if (section.children && section.children.length > 0) {
+        // Multiple parents, each with children
+        const childValues = {};
+        for (const child of section.children) {
+          const cr = extractChildSection($, el, child, context.origin);
+          if (cr && cr.type === "value" && cr.text !== null && cr.text !== undefined) {
+            childValues[child.name] = cr.text;
+          }
         }
-      });
+        if (Object.keys(childValues).length > 0) {
+          items.push(childValues);
+        }
+      } else {
+        // Multiple parents, no children
+        const value = extractValue(el, section, context.origin);
+        if (value && value.trim()) {
+          items.push(value.trim());
+        }
+      }
     }
 
-    if (results.length === 0) {
-      return `No results found on ${engine}. The engine may have shown a captcha, or the query returned nothing.`;
+    if (section.children && section.children.length > 0) {
+      return { section, type: "children-multiple", items };
+    } else {
+      return { section, type: "list", items };
     }
+  } else {
+    // Single parent
+    const el = elements.eq(0);
 
-    return (
-      `Found ${results.length} search results on ${engine}:\n\n` +
-      results
-        .map(
-          (r) =>
-            `[${r.position}] ${r.title}\n    URL: ${r.link}\n    Summary: ${r.snippet}`,
-        )
-        .join("\n\n")
-    );
-  } finally {
-    await page.close();
-    await context.close();
+    if (section.children && section.children.length > 0) {
+      // Single parent with children — parent format ignored
+      const childValues = {};
+      for (const child of section.children) {
+        const cr = extractChildSection($, el, child, context.origin);
+        if (cr && cr.type === "value" && cr.text !== null && cr.text !== undefined) {
+          childValues[child.name] = cr.text;
+        }
+      }
+      return { section, type: "children", items: childValues };
+    } else {
+      const value = extractValue(el, section, context.origin);
+      return { section, type: "value", text: value };
+    }
   }
 }
 
-async function executeFetch(url, format, startIndex, maxLength, blockMedia) {
-  const browser = await browserManager.getBrowser();
-  const context = await browser.newContext();
-  const page = await context.newPage();
+function extractTemplate($, template, context) {
+  const results = [];
 
-  try {
-    if (blockMedia) {
-      await page.route("**/*", (route) => {
-        const type = route.request().resourceType();
-        if (["image", "media", "font"].includes(type)) {
-          route.abort();
+  for (const section of template.sections) {
+    try {
+      const result = extractSection($, section, context);
+      if (result !== null) {
+        results.push(result);
+      }
+    } catch (err) {
+      if (
+        err.message &&
+        err.message.includes("Required section")
+      ) {
+        throw err;
+      }
+      // Non-required failures are silently skipped
+    }
+  }
+
+  return results;
+}
+
+// === COMPOSITION: WEBFETCH ===============================================
+
+function isCommentStyle(result) {
+  if (!result.items || result.items.length === 0) return false;
+  const first = result.items[0];
+  const keys = Object.keys(first).map((k) => k.toLowerCase());
+  return (
+    (keys.includes("author") && (keys.includes("comment") || keys.includes("body"))) ||
+    (keys.includes("user") && (keys.includes("comment") || keys.includes("body")))
+  );
+}
+
+function composeSections(extracted, template, startIndex, maxLength) {
+  const parts = [];
+
+  for (const result of extracted) {
+    if (result.type === "value") {
+      const text = result.text;
+      if (text && String(text).trim()) {
+        parts.push(`## ${result.section.name}\n\n${String(text).trim()}`);
+      }
+    } else if (result.type === "list") {
+      if (result.items && result.items.length > 0) {
+        const listText = result.items.map((item) => `- ${item}`).join("\n");
+        parts.push(`## ${result.section.name}\n\n${listText}`);
+      }
+    } else if (result.type === "children") {
+      if (result.items && Object.keys(result.items).length > 0) {
+        for (const [childName, value] of Object.entries(result.items)) {
+          if (value && String(value).trim()) {
+            parts.push(`## ${childName}\n\n${String(value).trim()}`);
+          }
+        }
+      }
+    } else if (result.type === "children-multiple") {
+      if (result.items && result.items.length > 0) {
+        if (isCommentStyle(result)) {
+          const commentParts = [];
+          for (const item of result.items) {
+            const author =
+              item["Author"] || item["author"] || item["User"] || item["user"] || "";
+            const comment =
+              item["Comment"] || item["Body"] || item["comment"] || item["body"] || "";
+            if (author) {
+              commentParts.push(`**${author}:**\n\n${comment}`);
+            } else if (comment) {
+              commentParts.push(comment);
+            }
+          }
+          if (commentParts.length > 0) {
+            parts.push(
+              `## ${result.section.name}\n\n${commentParts.join("\n\n---\n\n")}`,
+            );
+          }
         } else {
-          route.continue();
+          const itemParts = [];
+          for (const item of result.items) {
+            const lines = [];
+            for (const [key, value] of Object.entries(item)) {
+              if (value && String(value).trim()) {
+                lines.push(`    ${key}: ${String(value).trim()}`);
+              }
+            }
+            if (lines.length > 0) itemParts.push(lines.join("\n"));
+          }
+          if (itemParts.length > 0) {
+            parts.push(
+              `## ${result.section.name}\n\n${itemParts.join("\n\n")}`,
+            );
+          }
         }
-      });
+      }
     }
+  }
 
-    try {
-      await page.goto(url, { waitUntil: "networkidle", timeout: 15000 });
-    } catch (navError) {
-      // Allow partial rendering on timeout
+  if (parts.length === 0) {
+    return "(No content extracted from this page.)";
+  }
+
+  const full = parts.join("\n\n---\n\n");
+  const totalLength = full.length;
+  const paginated = full.substring(startIndex, startIndex + maxLength);
+
+  const templateName = template ? template.name : "auto";
+  let metadata = `\n\n---\n[webfetch: template="${templateName}", showing characters ${startIndex} to ${startIndex + paginated.length} of ${totalLength} total.`;
+  if (startIndex + maxLength < totalLength) {
+    metadata += ` Use start_index=${startIndex + maxLength} to read more.`;
+  }
+  metadata += `]`;
+
+  return paginated + metadata;
+}
+
+// === COMPOSITION: WEBSEARCH ==============================================
+
+function composeSearchResults(extracted) {
+  // Find the search results section (first children-multiple)
+  const searchSection = extracted.find((r) => r.type === "children-multiple");
+
+  if (!searchSection || !searchSection.items || searchSection.items.length === 0) {
+    // Fall back to section-based output
+    return composeSections(extracted, null, 0, Infinity);
+  }
+
+  const items = searchSection.items;
+  const parts = [];
+
+  for (let i = 0; i < items.length; i++) {
+    const item = items[i];
+    const num = i + 1;
+
+    const title =
+      item["Title"] || item["title"] || Object.values(item)[0] || "";
+    const url =
+      item["URL"] || item["url"] || item["Url"] || "";
+    const snippet =
+      item["Snippet"] || item["snippet"] || "";
+
+    // Filter out non-http URLs and google internal links
+    let cleanUrl = url;
+    if (cleanUrl && !cleanUrl.startsWith("http")) {
+      cleanUrl = ""; // Skip internal/non-web URLs
+    }
+    if (
+      cleanUrl &&
+      (cleanUrl.includes("google.com/search") ||
+        cleanUrl.includes("support.google.com"))
+    ) {
+      cleanUrl = ""; // Skip google internal links
     }
 
-    const pageContent = await page.content();
-    let finalContent = "";
+    if (!title) continue;
 
-    if (format === "raw_html") {
-      finalContent = pageContent;
-    } else {
-      const $ = cheerio.load(pageContent);
+    const lines = [`[${num}] ${title}`];
+    if (cleanUrl) lines.push(`    URL: ${cleanUrl}`);
+    if (snippet) lines.push(`    Snippet: ${snippet}`);
 
-      $(
-        "script, style, nav, header, footer, noscript, iframe, svg, aside, .advertisement, img, picture, video, audio, canvas, map, area, dialog",
-      ).remove();
-      $("*")
-        .removeAttr("style")
-        .each((i, el) => {
-          const src = $(el).attr("src");
-          if (src && src.startsWith("data:image")) $(el).removeAttr("src");
-        });
+    parts.push(lines.join("\n"));
+  }
 
-      if (format === "clean_html") {
-        finalContent = $.html();
-      } else {
-        const turndownService = new TurndownService({
-          headingStyle: "atx",
-          codeBlockStyle: "fenced",
-        });
-        finalContent = turndownService
-          .turndown($.html())
-          .replace(/\n{3,}/g, "\n\n")
-          .trim();
-      }
+  if (parts.length === 0) {
+    return "(No content extracted from this page.)";
+  }
+
+  return `## ${searchSection.section.name}\n\n${parts.join("\n\n")}`;
+}
+
+// === GENERIC FALLBACK ====================================================
+
+function genericFallback($, startIndex, maxLength) {
+  applyRemove($, null);
+
+  const bodyHtml = $("body").html() || "";
+  let markdown = turndown
+    .turndown(bodyHtml)
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+
+  if (!markdown || markdown.trim().length === 0) {
+    return "(No content extracted from this page.)";
+  }
+
+  const totalLength = markdown.length;
+  const paginated = markdown.substring(startIndex, startIndex + maxLength);
+
+  let metadata = `\n\n---\n[webfetch: template="auto (fallback)", showing characters ${startIndex} to ${startIndex + paginated.length} of ${totalLength} total.`;
+  if (startIndex + maxLength < totalLength) {
+    metadata += ` Use start_index=${startIndex + maxLength} to read more.`;
+  }
+  metadata += `]`;
+
+  return paginated + metadata;
+}
+
+// === SEARCH TEMPLATE RESOLUTION ==========================================
+
+function resolveSearchTemplate(engine, query, region, safeSearch) {
+  const templateName = resolveEngineToTemplateName(engine);
+
+  let template;
+  if (templateName.startsWith("{")) {
+    try {
+      template = JSON.parse(templateName);
+    } catch (e) {
+      throw new Error(`Invalid inline JSON template: ${e.message}`);
     }
+  } else {
+    template = getTemplateByName(templateName);
+  }
 
-    const totalLength = finalContent.length;
-    let paginatedText = finalContent.substring(
-      startIndex,
-      startIndex + maxLength,
+  if (!template.url_template) {
+    throw new Error(
+      `Template '${template.name}' is not a search template (no url_template).`,
     );
+  }
 
-    let metadata = `\n\n---\n[Document Info: Showing characters ${startIndex} to ${startIndex + paginatedText.length} of ${totalLength} total.`;
-    if (startIndex + maxLength < totalLength) {
-      metadata += ` Use start_index=${startIndex + maxLength} to read more.`;
-    }
-    metadata += `]`;
+  const params = mapSearchParams(engine, query, region, safeSearch);
+  let url = resolveUrlTemplate(template, params);
 
-    return paginatedText + metadata;
-  } finally {
-    await page.close();
-    await context.close();
+  // Google safe_search: append safe=active to URL
+  if (
+    (engine === "google" || templateName === "google-search") &&
+    safeSearch === true
+  ) {
+    url += "&safe=active";
   }
+
+  return { template, url };
 }
 
-const server = new McpServer({ name: "searchfetch", version: "2.0.0" });
+// === MCP SERVER & TOOLS ==================================================
+
+const server = new McpServer({ name: "searchfetch", version: "3.0.0" });
+
+// --- websearch tool ---
 
 server.registerTool(
   "websearch",
@@ -296,38 +933,67 @@ server.registerTool(
     inputSchema: z.object({
       query: z.string().describe("The search query string."),
       engine: z
-        .enum(["duckduckgo", "google"])
+        .string()
         .default("duckduckgo")
         .describe(
           "Search engine to use. Can be 'duckduckgo' or 'google'. Default is 'duckduckgo'.",
         ),
-      max_results: z
-        .number()
-        .default(10)
-        .describe("Maximum number of results to return. Default is 10."),
       region: z
         .string()
-        .default("wt-wt")
+        .nullable()
+        .default(null)
         .describe(
-          "Region and language code to localize search results (e.g., 'us-en', 'uk-en', 'de-de'). For DuckDuckGo it maps directly. For Google, 'us' is country code and 'en' is language. Default is 'wt-wt' (global/US English).",
+          "Region and language code to localize search results (e.g., 'us-en', 'uk-en', 'de-de'). For DuckDuckGo it maps directly. For Google, 'us' is country code and 'en' is language. Default is null (uses template default).",
         ),
       safe_search: z
-        .string()
-        .default("-1")
+        .boolean()
+        .nullable()
+        .default(null)
+        .describe(
+          "Enable safe search filtering. null = use template default. Applies to both DuckDuckGo and Google.",
+        ),
+      max_results: z
+        .number()
+        .default(10)
+        .describe("Maximum number of search results to return. Default is 10."),
+      block_media: z
+        .boolean()
+        .default(true)
         .describe(
-          "Safe search filtering mode. '-1' for Moderate, '1' for Strict, '-2' for Off. Default is '-1'. Note: Only applies to DuckDuckGo.",
+          "Block images, videos, and fonts entirely at the network layer. Default is true.",
         ),
     }),
   },
-  async ({ query, max_results, region, safe_search, engine }) => {
+  async ({ query, engine, region, safe_search, max_results, block_media }) => {
     try {
-      const result = await executeSearch(
+      // 1. Resolve search template (+ url_params mapping + url building)
+      const { template, url } = resolveSearchTemplate(
+        engine,
         query,
-        max_results,
         region,
         safe_search,
-        engine,
       );
+
+      // 2. Fetch
+      const html = await fetchHtmlWithRetry(url, template, block_media);
+
+      // 3. Extract
+      const $ = cheerio.load(html);
+      applyRemove($, template);
+
+      const pageOrigin = new URL(url).origin;
+      const context = {
+        origin: pageOrigin,
+        isWebsearch: true,
+        maxResultsOverride: max_results,
+        _maxResultsConsumed: false,
+      };
+
+      const extracted = extractTemplate($, template, context);
+
+      // 4. Compose
+      const result = composeSearchResults(extracted);
+
       return { content: [{ type: "text", text: result }] };
     } catch (err) {
       return {
@@ -338,6 +1004,8 @@ server.registerTool(
   },
 );
 
+// --- webfetch tool ---
+
 server.registerTool(
   "webfetch",
   {
@@ -345,22 +1013,20 @@ server.registerTool(
     description:
       "Fetch and extract the main text content from any webpage. Fully executes JavaScript to load React/SPAs and aggressively strips images/media (including base64) to save context tokens.",
     inputSchema: z.object({
-      url: z
-        .url()
-        .describe(
-          "The full URL of the webpage to fetch (must start with http/https).",
-        ),
-      format: z
-        .enum(["markdown", "clean_html", "raw_html"])
-        .default("markdown")
+      url: z.string().describe(
+        "The full URL of the webpage to fetch (must start with http/https).",
+      ),
+      template: z
+        .string()
+        .default("auto")
         .describe(
-          "Output format. Set to 'markdown', 'clean_html', or 'raw_html'. Default is 'markdown' (highly recommended to save context tokens).",
+          "Template to use: 'auto' (auto-detect from URL), a built-in name, or inline JSON.",
         ),
       start_index: z
         .number()
         .default(0)
         .describe(
-          "Character offset to start reading from for pagination. Use this if a document is too large to fit in the context window. Default is 0.",
+          "Character offset for pagination. Default: 0.",
         ),
       max_length: z
         .number()
@@ -372,20 +1038,50 @@ server.registerTool(
         .boolean()
         .default(true)
         .describe(
-          "Block images, videos, and fonts entirely at the network layer to drastically speed up page loads and dodge tracking pixels. Default is true.",
+          "Block images, videos, and fonts entirely at the network layer. Default is true.",
         ),
     }),
   },
-  async ({ url, format, start_index, max_length, block_media }) => {
+  async ({ url, template: templateParam, start_index, max_length, block_media }) => {
     try {
-      const result = await executeFetch(
-        url,
-        format,
-        start_index,
-        max_length,
-        block_media,
-      );
-      return { content: [{ type: "text", text: result }] };
+      // 1. Resolve template
+      let template;
+
+      if (templateParam.startsWith("{")) {
+        try {
+          template = JSON.parse(templateParam);
+        } catch (e) {
+          throw new Error(`Invalid inline JSON template: ${e.message}`);
+        }
+      } else if (templateParam === "auto") {
+        template = detectTemplateByUrl(url);
+      } else {
+        template = getTemplateByName(templateParam);
+      }
+
+      // 2. Fetch
+      const html = await fetchHtmlWithRetry(url, template, block_media);
+
+      // 3. Extract and compose
+      const $ = cheerio.load(html);
+
+      if (template) {
+        applyRemove($, template);
+
+        const pageOrigin = new URL(url).origin;
+        const context = {
+          origin: pageOrigin,
+          isWebsearch: false,
+        };
+
+        const extracted = extractTemplate($, template, context);
+        const result = composeSections(extracted, template, start_index, max_length);
+        return { content: [{ type: "text", text: result }] };
+      } else {
+        // Generic fallback
+        const result = genericFallback($, start_index, max_length);
+        return { content: [{ type: "text", text: result }] };
+      }
     } catch (err) {
       return {
         content: [{ type: "text", text: `Fetch Error: ${err.message}` }],
@@ -395,6 +1091,8 @@ server.registerTool(
   },
 );
 
+// === MAIN =================================================================
+
 async function main() {
   await ensureBinary();
   process.stdout.write = originalStdoutWrite;