searchfetch 1.0.2 → 3.0.0

package/index.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 
+// === STDOUT/STDERR REDIRECTION ===========================================
 const originalStdoutWrite = process.stdout.write.bind(process.stdout);
 process.stdout.write = (chunk, encoding, callback) => {
   return process.stderr.write(chunk, encoding, callback);
@@ -7,6 +8,10 @@ process.stdout.write = (chunk, encoding, callback) => {
 console.log = (...args) => console.error(...args);
 console.info = (...args) => console.error(...args);
 
+// === IMPORTS =============================================================
+import { readdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
@@ -14,37 +19,47 @@ import { launch, ensureBinary } from "cloakbrowser";
 import * as cheerio from "cheerio";
 import TurndownService from "turndown";
 
-const logger = {
-  info: (msg) => console.error(`[INFO] ${msg}`),
-  warn: (msg) => console.error(`[WARN] ${msg}`),
-  error: (msg, err) => console.error(`[ERROR] ${msg}`, err || ""),
-};
-
-// ==========================================
-// BROWSER LIFECYCLE MANAGEMENT
-// ==========================================
+// === BROWSER MANAGER =====================================================
 class BrowserManager {
   constructor() {
     this.browser = null;
+    this.launchPromise = null;
   }
 
   async getBrowser() {
-    if (!this.browser) {
-      logger.info("Launching stealth CloakBrowser instance...");
-      this.browser = await launch({
-        headless: true,
-        humanize: true, // Native C++ bot-bypass patches + human behavior
-        args: ["--disable-blink-features=AutomationControlled", "--no-sandbox"],
+    if (this.browser && this.browser.isConnected()) return this.browser;
+    if (this.launchPromise) return this.launchPromise;
+
+    this.launchPromise = launch({
+      headless: true,
+      humanize: true,
+      args: [
+        "--disable-blink-features=AutomationControlled",
+        "--no-sandbox",
+        "--disable-dev-shm-usage",
+      ],
+    })
+      .then((browser) => {
+        this.browser = browser;
+        this.browser.on("disconnected", () => {
+          this.browser = null;
+        });
+        return browser;
+      })
+      .catch((err) => {
+        throw err;
+      })
+      .finally(() => {
+        this.launchPromise = null;
       });
-    }
-    return this.browser;
+
+    return this.launchPromise;
   }
 
   async close() {
     if (this.browser) {
       await this.browser.close();
       this.browser = null;
-      logger.info("Browser instance securely closed.");
     }
   }
 }
@@ -52,358 +67,1039 @@ class BrowserManager {
 const browserManager = new BrowserManager();
 
 const cleanup = async () => {
-  logger.info("Received termination signal. Shutting down browser...");
   await browserManager.close();
   process.exit(0);
 };
 process.on("SIGINT", cleanup);
 process.on("SIGTERM", cleanup);
 
-// ==========================================
-// CORE LOGIC: SEARCH & FETCH
-// ==========================================
+// === TURNDOWN ============================================================
+const turndown = new TurndownService({
+  headingStyle: "atx",
+  codeBlockStyle: "fenced",
+  emDelimiter: "*",
+});
+
+// === BUILT-IN TEMPLATES (loaded from templates/*.json) ====================
 
-async function executeSearch(query, maxResults, region, safeSearch, engine) {
-  logger.info(
-    `Searching ${engine.toUpperCase()} via Stealth Browser for: "${query}"`,
-  );
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const TEMPLATES_DIR = join(__dirname, "templates");
+
+function loadBuiltinTemplates() {
+  let files;
+  try {
+    files = readdirSync(TEMPLATES_DIR);
+  } catch (err) {
+    throw new Error(
+      `Cannot read templates directory '${TEMPLATES_DIR}': ${err.message}`,
+    );
+  }
+
+  const jsonFiles = files
+    .filter((f) => f.endsWith(".json"))
+    .sort();
+
+  if (jsonFiles.length === 0) {
+    throw new Error(
+      `No template JSON files found in '${TEMPLATES_DIR}'`,
+    );
+  }
+
+  const templates = [];
+  for (const file of jsonFiles) {
+    const filePath = join(TEMPLATES_DIR, file);
+    const content = readFileSync(filePath, "utf-8");
+    let template;
+    try {
+      template = JSON.parse(content);
+    } catch (err) {
+      throw new Error(
+        `Invalid JSON in template file '${filePath}': ${err.message}`,
+      );
+    }
+    if (!template.name || typeof template.name !== "string") {
+      throw new Error(
+        `Template file '${filePath}' is missing a valid "name" field`,
+      );
+    }
+    templates.push(template);
+  }
+
+  // Sort by "order" field for deterministic URL-pattern matching
+  templates.sort((a, b) => (a.order ?? 999) - (b.order ?? 999));
+
+  return templates;
+}
+
+const BUILTIN_TEMPLATES = loadBuiltinTemplates();
+
+// === TEMPLATE LOOKUP =====================================================
+const TEMPLATE_MAP = new Map();
+for (const t of BUILTIN_TEMPLATES) {
+  TEMPLATE_MAP.set(t.name, t);
+}
+
+function getTemplateByName(name) {
+  const t = TEMPLATE_MAP.get(name);
+  if (!t) {
+    const names = [...TEMPLATE_MAP.keys()].join(", ");
+    throw new Error(
+      `Unknown template '${name}'. Available: ${names}`,
+    );
+  }
+  return t;
+}
+
+function detectTemplateByUrl(url) {
+  for (const template of BUILTIN_TEMPLATES) {
+    if (!template.url_patterns) continue;
+    for (const pattern of template.url_patterns) {
+      try {
+        if (new RegExp(pattern).test(url)) {
+          return template;
+        }
+      } catch (_) {
+        // Skip invalid regex
+      }
+    }
+  }
+  return null;
+}
+
+// === URL TEMPLATE RESOLUTION =============================================
+
+function resolveUrlTemplate(template, providedParams) {
+  const urlParams = template.url_params || {};
+  let url = template.url_template;
+  if (!url) return null;
+
+  let match;
+  const re = /\{(\w+)\}/g;
+  while ((match = re.exec(url)) !== null) {
+    const name = match[1];
+    const def = urlParams[name] || {};
+
+    let value;
+    if (
+      name in providedParams &&
+      providedParams[name] !== null &&
+      providedParams[name] !== undefined
+    ) {
+      value = String(providedParams[name]);
+    } else if (def.default !== undefined) {
+      value = String(def.default);
+    } else if (def.required) {
+      throw new Error(
+        `Required URL parameter '${name}' not provided for template '${template.name}'.`,
+      );
+    } else {
+      value = "";
+    }
+
+    if (def.encode === "url") {
+      value = encodeURIComponent(value);
+    }
+
+    url = url.replace(match[0], value);
+  }
+
+  // Remove any remaining unreplaced placeholders
+  url = url.replace(/\{\w+\}/g, "").replace(/&{2,}/g, "&").replace(/\?&/, "?");
+
+  return url;
+}
+
+// === SEARCH PARAM MAPPING ================================================
+
+function resolveEngineToTemplateName(engine) {
+  if (engine === "duckduckgo") return "duckduckgo-search";
+  if (engine === "google") return "google-search";
+  return engine;
+}
+
+function mapSearchParams(engine, query, region, safeSearch) {
+  const params = { query };
+  const resolved = resolveEngineToTemplateName(engine);
+
+  if (resolved === "duckduckgo-search") {
+    if (region !== null && region !== undefined) {
+      params.kl = region;
+    }
+    if (safeSearch === true) {
+      params.kp = "1";
+    } else if (safeSearch === false) {
+      params.kp = "-2";
+    }
+  } else if (resolved === "google-search") {
+    if (region !== null && region !== undefined) {
+      const parts = region.split("-");
+      params.hl = parts[0];
+      params.gl = parts.length > 1 ? parts[1] : parts[0];
+    }
+  }
+
+  return params;
+}
+
+// === FETCH ===============================================================
+
+function isAccessDenied($) {
+  const title = ($("title").text() || "").toLowerCase();
+  const bodyText = ($("body").text() || "").replace(/\s+/g, " ").trim().toLowerCase();
+
+  const titleDenyPatterns = [
+    "captcha",
+    "are you a robot",
+    "access denied",
+    "blocked",
+    "forbidden",
+    "unusual traffic",
+    "sorry, you have been blocked",
+    "verify you are human",
+    "one more step",
+    "security check",
+    "ddos protection",
+    "cloudflare",
+  ];
+
+  if (titleDenyPatterns.some((pattern) => title.includes(pattern))) return true;
+
+  const bodyDenyPatterns = [
+    "to continue, please type the characters",
+    "our systems have detected unusual traffic",
+    "verify you are human",
+    "are you a robot",
+    "sorry, you have been blocked",
+    "access denied",
+  ];
+
+  if (bodyText.length < 1200 && bodyDenyPatterns.some((pattern) => bodyText.includes(pattern))) return true;
 
+  return false;
+}
+
+async function fetchHtml(url, template, blockMedia) {
   const browser = await browserManager.getBrowser();
   const context = await browser.newContext();
 
-  // Inject Google Consent cookie to universally bypass GDPR popups blocking the DOM
-  await context.addCookies([
-    {
-      name: "CONSENT",
-      value: "YES+cb.20250101-01-p0.en+FX+999",
-      domain: ".google.com",
-      path: "/",
-    },
-  ]);
+  try {
+    // Pre-load cookies from template
+    if (template && template.cookies && template.cookies.length > 0) {
+      await context.addCookies(template.cookies);
+    }
 
-  const page = await context.newPage();
+    const page = await context.newPage();
 
-  try {
-    // Optimization: Block heavy/unnecessary resources to make searches lightning fast
-    await page.route("**/*", (route) => {
-      const type = route.request().resourceType();
-      if (["image", "media", "font", "stylesheet"].includes(type)) {
-        route.abort();
-      } else {
-        route.continue();
+    try {
+      // Route blocked resource types
+      if (blockMedia) {
+        const blockedTypes =
+          template && template.block_resources
+            ? template.block_resources
+            : ["image", "media", "font"];
+
+        if (blockedTypes.length > 0) {
+          await page.route("**/*", (route) => {
+            const type = route.request().resourceType();
+            if (blockedTypes.includes(type)) {
+              route.abort();
+            } else {
+              route.continue();
+            }
+          });
+        }
       }
-    });
 
-    const results = [];
-    let searchUrl = "";
+      let response;
+      try {
+        response = await page.goto(url, {
+          waitUntil: "networkidle",
+          timeout: 15000,
+        });
+      } catch (_navError) {
+        // Allow partial rendering on timeout
+      }
 
-    if (engine === "google") {
-      searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&hl=en&gl=us`;
-    } else {
-      searchUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}&kl=${encodeURIComponent(region)}&kp=${encodeURIComponent(safeSearch)}`;
+      // Check HTTP status for access failures
+      if (response) {
+        const status = response.status();
+        if ([401, 403, 429].includes(status)) {
+          throw new Error(
+            `Access denied: HTTP ${status} when fetching ${url}`,
+          );
+        }
+      }
+
+      const pageContent = await page.content();
+
+      // Check for CAPTCHA / access-denied pages
+      const $ = cheerio.load(pageContent);
+      if (isAccessDenied($)) {
+        throw new Error(
+          `Access denied: CAPTCHA or block page detected at ${url}. The site is blocking automated access.`,
+        );
+      }
+
+      return pageContent;
+    } finally {
+      await page.close();
     }
+  } finally {
+    await context.close();
+  }
+}
 
+async function fetchHtmlWithRetry(url, template, blockMedia) {
+  let lastError;
+  for (let attempt = 0; attempt < 2; attempt++) {
     try {
-      // Use networkidle to ensure JavaScript fully renders organic results or follows hidden redirects
-      await page.goto(searchUrl, { waitUntil: "networkidle", timeout: 15000 });
-    } catch (e) {
-      if (e.name === "TimeoutError") {
-        logger.warn(`Network idle timeout on search. Extracting loaded DOM...`);
-      } else {
-        throw e;
+      return await fetchHtml(url, template, blockMedia);
+    } catch (err) {
+      lastError = err;
+      if (
+        attempt === 0 &&
+        (err.message.includes("net::") ||
+          err.message.includes("ERR_") ||
+          err.message.includes("Navigation failed"))
+      ) {
+        // Network error — retry once
+        continue;
       }
+      throw err;
     }
+  }
+  throw lastError;
+}
 
-    const pageContent = await page.content();
-    const $ = cheerio.load(pageContent);
+// === HTML CLEANUP ========================================================
 
-    if (engine === "google") {
-      // Google's core organic result selector
-      $("div.g").each((i, el) => {
-        if (results.length >= maxResults) return;
+const DEFAULT_REMOVE_SELECTORS = [
+  "script", "style", "svg", "nav", "footer", "noscript", "iframe",
+  ".advertisement",
+];
 
-        const titleEl = $(el).find("h3").first();
-        const linkEl = $(el).find("a").first();
-        if (!titleEl.length || !linkEl.length) return;
+function applyRemove($, template) {
+  const removeSelectors =
+    template && template.remove && template.remove.length > 0
+      ? template.remove
+      : DEFAULT_REMOVE_SELECTORS;
 
-        const title = titleEl.text().trim();
-        let link = linkEl.attr("href") || "";
+  for (const selector of removeSelectors) {
+    try {
+      $(selector).remove();
+    } catch (_) {
+      // Skip invalid selectors
+    }
+  }
 
-        // Handle Google relative redirect links
-        if (link.startsWith("/url?q=")) {
-          try {
-            link = decodeURIComponent(link.split("/url?q=")[1].split("&")[0]);
-          } catch (e) {}
-        }
+  // Strip style attributes and data:image src
+  $("[style]").removeAttr("style");
+  $("*").each((_i, el) => {
+    const src = $(el).attr("src");
+    if (src && src.startsWith("data:image")) {
+      $(el).removeAttr("src");
+    }
+  });
+}
 
-        // Isolate snippet text safely
-        const cloned = $(el).clone();
-        cloned.find("h3, a, script, style, cite").remove();
-        const snippet = cloned.text().replace(/\s+/g, " ").trim();
+// === EXTRACTION ENGINE ===================================================
 
-        if (title && link && link.startsWith("http")) {
-          results.push({ position: results.length + 1, title, link, snippet });
-        }
-      });
-    } else {
-      // DuckDuckGo selector
-      $(".result").each((i, el) => {
-        if (results.length >= maxResults) return;
+/**
+ * Find elements matching selector, scoped to $parent.
+ * Search order: descendants → closest ancestor → ancestor subtrees (up to 4 levels).
+ */
+function findScoped($parent, selector) {
+  if (!selector || selector.trim() === "") {
+    return $parent;
+  }
+
+  // 1. Descendants
+  let result = $parent.find(selector);
+  if (result.length > 0) return result;
+
+  // 2. Closest ancestor matching selector
+  result = $parent.closest(selector);
+  if (result.length > 0) return result;
+
+  // 3. Ancestor subtrees (up to 4 levels up)
+  let ancestor = $parent.parent();
+  for (let i = 0; i < 4 && ancestor.length > 0; i++) {
+    result = ancestor.find(selector);
+    if (result.length > 0) return result;
+    ancestor = ancestor.parent();
+  }
+
+  return $parent.find("__nonexistent__");
+}
 
-        const titleEl = $(el).find(".result__title a");
-        const snippetEl = $(el).find(".result__snippet");
-        if (!titleEl.length) return;
+/**
+ * Try comma-separated selectors in order; first match wins.
+ */
+function findFirstMatch($parent, selectorStr) {
+  if (!selectorStr || selectorStr.trim() === "") {
+    return $parent;
+  }
+
+  const selectors = selectorStr
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
 
-        const title = titleEl.text().trim();
-        let link = titleEl.attr("href") || "";
+  for (const sel of selectors) {
+    const matches = findScoped($parent, sel);
+    if (matches.length > 0) return matches;
+  }
+
+  return $parent.find("__nonexistent__");
+}
 
-        if (link.includes("/l/?uddg=")) {
+/**
+ * Resolve top-level elements for a section (document-wide with fallback).
+ */
+function resolveTopElements($, selectorStr) {
+  if (!selectorStr || selectorStr.trim() === "") {
+    return $("body");
+  }
+
+  const selectors = selectorStr
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
+
+  for (const sel of selectors) {
+    try {
+      const matches = $(sel);
+      if (matches.length > 0) return matches;
+    } catch (_) {
+      // Skip invalid selectors
+    }
+  }
+
+  return $();
+}
+
+// === TRANSFORMS ==========================================================
+
+function applyTransform(value, transform, origin) {
+  const transforms = Array.isArray(transform) ? transform : [transform];
+  let result = value;
+
+  for (const t of transforms) {
+    if (!result) continue;
+    switch (t) {
+      case "strip":
+        result = result.trim();
+        break;
+
+      case "decode_google_url":
+        if (result.startsWith("/url?q=")) {
           try {
-            const urlParams = new URLSearchParams(link.split("?")[1]);
-            link = decodeURIComponent(urlParams.get("uddg") || link);
-          } catch (e) {}
+            const urlPart = result.split("/url?q=")[1].split("&")[0];
+            result = decodeURIComponent(urlPart);
+          } catch (_) {
+            // Leave as-is
+          }
         }
+        break;
 
-        const snippet = snippetEl.text().replace(/\s+/g, " ").trim();
-        if (title && link && link.startsWith("http")) {
-          results.push({ position: results.length + 1, title, link, snippet });
+      case "decode_ddg_url":
+        if (result.includes("/l/?uddg=")) {
+          try {
+            const queryString = result.split("?")[1] || "";
+            const params = new URLSearchParams(queryString);
+            const uddg = params.get("uddg");
+            if (uddg) result = decodeURIComponent(uddg);
+          } catch (_) {
+            // Leave as-is
+          }
         }
-      });
+        break;
+
+      case "json_parse":
+        try {
+          result = JSON.stringify(JSON.parse(result), null, 2);
+        } catch (_) {
+          // Leave as-is
+        }
+        break;
+
+      case "resolve_href":
+        if (origin && result.startsWith("/") && !result.startsWith("//")) {
+          try {
+            result = new URL(result, origin).href;
+          } catch (_) {
+            // Leave as-is
+          }
+        }
+        break;
     }
+  }
+
+  return result;
+}
+
+// === EXTRACTION ==========================================================
+
+function extractValue($el, section, origin) {
+  let value;
 
-    if (results.length === 0) {
-      const pageText = $("body").text().replace(/\s+/g, " ").substring(0, 300);
-      logger.warn(`No results found. DOM Sample: ${pageText}`);
-      return `No results found on ${engine}. The search engine might be showing a captcha/consent screen, or the query returned nothing. Try rephrasing or switching engines.`;
+  switch (section.format) {
+    case "text":
+      value = $el.text().replace(/\s+/g, " ").trim();
+      break;
+
+    case "markdown": {
+      const html = $el.html() || "";
+      value = turndown
+        .turndown(html)
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+      break;
     }
 
-    return (
-      `Found ${results.length} search results on ${engine}:\n\n` +
-      results
-        .map(
-          (r) =>
-            `[${r.position}] ${r.title}\n    URL: ${r.link}\n    Summary: ${r.snippet}`,
-        )
-        .join("\n\n")
-    );
-  } finally {
-    await page.close();
-    await context.close();
+    case "attribute":
+      value = $el.attr(section.attribute) || "";
+      break;
+
+    case "html":
+      value = $el.html() || "";
+      break;
+
+    default:
+      value = $el.text().replace(/\s+/g, " ").trim();
   }
+
+  if (section.transform && value) {
+    value = applyTransform(value, section.transform, origin);
+  }
+
+  return value;
 }
 
-async function executeFetch(url, format, startIndex, maxLength, blockMedia) {
-  logger.info(`Fetching URL: ${url} | Format: ${format}`);
+/**
+ * Extract one child section, scoped to $parentEl.
+ * Returns { type: "value", text } or null.
+ */
+function extractChildSection($, $parentEl, section, origin) {
+  const elements = findFirstMatch($parentEl, section.selector);
 
-  const browser = await browserManager.getBrowser();
-  const context = await browser.newContext();
-  const page = await context.newPage();
+  if (!elements || elements.length === 0) {
+    if (section.required) {
+      throw new Error(
+        `Required section '${section.name}' not found on page.`,
+      );
+    }
+    return null;
+  }
 
-  try {
-    if (blockMedia) {
-      await page.route("**/*", (route) => {
-        const type = route.request().resourceType();
-        if (["image", "media", "font"].includes(type)) {
-          route.abort();
-        } else {
-          route.continue();
-        }
-      });
+  const el = elements.eq(0);
+  const value = extractValue(el, section, origin);
+  return { type: "value", text: value };
+}
+
+/**
+ * Extract a top-level section from the document.
+ * Returns a SectionResult or null.
+ */
+function extractSection($, section, context) {
+  const elements = resolveTopElements($, section.selector);
+
+  if (!elements || elements.length === 0) {
+    if (section.required) {
+      throw new Error(
+        `Required section '${section.name}' not found on page.`,
+      );
     }
+    return null;
+  }
 
-    try {
-      await page.goto(url, { waitUntil: "networkidle", timeout: 15000 });
-    } catch (navError) {
-      if (navError.name === "TimeoutError") {
-        logger.warn(
-          `Network idle timeout on ${url}. Extracting partial DOM...`,
-        );
+  // Determine limit
+  let limit = elements.length;
+  if (section.multiple && section.max_items) {
+    limit = Math.min(limit, section.max_items);
+  }
+  // Override max_items with max_results for first multiple+children section
+  if (
+    context.isWebsearch &&
+    context.maxResultsOverride &&
+    !context._maxResultsConsumed &&
+    section.multiple &&
+    section.children &&
+    section.children.length > 0
+  ) {
+    limit = Math.min(limit, context.maxResultsOverride);
+    context._maxResultsConsumed = true;
+  }
+
+  if (section.multiple) {
+    const items = [];
+
+    for (let i = 0; i < limit; i++) {
+      const el = elements.eq(i);
+
+      if (section.children && section.children.length > 0) {
+        // Multiple parents, each with children
+        const childValues = {};
+        for (const child of section.children) {
+          const cr = extractChildSection($, el, child, context.origin);
+          if (cr && cr.type === "value" && cr.text !== null && cr.text !== undefined) {
+            childValues[child.name] = cr.text;
+          }
+        }
+        if (Object.keys(childValues).length > 0) {
+          items.push(childValues);
+        }
       } else {
-        throw navError;
+        // Multiple parents, no children
+        const value = extractValue(el, section, context.origin);
+        if (value && value.trim()) {
+          items.push(value.trim());
+        }
       }
     }
 
-    const pageContent = await page.content();
-    let finalContent = "";
+    if (section.children && section.children.length > 0) {
+      return { section, type: "children-multiple", items };
+    } else {
+      return { section, type: "list", items };
+    }
+  } else {
+    // Single parent
+    const el = elements.eq(0);
 
-    if (format === "raw_html") {
-      finalContent = pageContent;
+    if (section.children && section.children.length > 0) {
+      // Single parent with children — parent format ignored
+      const childValues = {};
+      for (const child of section.children) {
+        const cr = extractChildSection($, el, child, context.origin);
+        if (cr && cr.type === "value" && cr.text !== null && cr.text !== undefined) {
+          childValues[child.name] = cr.text;
+        }
+      }
+      return { section, type: "children", items: childValues };
     } else {
-      const $ = cheerio.load(pageContent);
+      const value = extractValue(el, section, context.origin);
+      return { section, type: "value", text: value };
+    }
+  }
+}
+
+function extractTemplate($, template, context) {
+  const results = [];
 
-      // 🚀 NUCLEAR OPTION FOR BASE64 AND TOKENS 🚀
-      // Physically scrub out all tags that harbor base64 strings or waste tokens
-      $(
-        "script, style, nav, header, footer, noscript, iframe, svg, aside, .advertisement, img, picture, video, audio, canvas, map, area, dialog",
-      ).remove();
+  for (const section of template.sections) {
+    try {
+      const result = extractSection($, section, context);
+      if (result !== null) {
+        results.push(result);
+      }
+    } catch (err) {
+      if (
+        err.message &&
+        err.message.includes("Required section")
+      ) {
+        throw err;
+      }
+      // Non-required failures are silently skipped
+    }
+  }
 
-      // Remove inline styles from EVERY element to prevent background-image base64 leaks
-      $("*").removeAttr("style");
+  return results;
+}
 
-      // Remove data URIs anywhere else in the document
-      $("*").each((i, el) => {
-        const src = $(el).attr("src");
-        if (src && src.startsWith("data:image")) $(el).removeAttr("src");
-      });
+// === COMPOSITION: WEBFETCH ===============================================
 
-      if (format === "clean_html") {
-        finalContent = $.html();
-      } else if (format === "markdown") {
-        const turndownService = new TurndownService({
-          headingStyle: "atx",
-          codeBlockStyle: "fenced",
-        });
-        finalContent = turndownService.turndown($.html());
-        finalContent = finalContent.replace(/\n{3,}/g, "\n\n").trim();
+function isCommentStyle(result) {
+  if (!result.items || result.items.length === 0) return false;
+  const first = result.items[0];
+  const keys = Object.keys(first).map((k) => k.toLowerCase());
+  return (
+    (keys.includes("author") && (keys.includes("comment") || keys.includes("body"))) ||
+    (keys.includes("user") && (keys.includes("comment") || keys.includes("body")))
+  );
+}
+
+function composeSections(extracted, template, startIndex, maxLength) {
+  const parts = [];
+
+  for (const result of extracted) {
+    if (result.type === "value") {
+      const text = result.text;
+      if (text && String(text).trim()) {
+        parts.push(`## ${result.section.name}\n\n${String(text).trim()}`);
+      }
+    } else if (result.type === "list") {
+      if (result.items && result.items.length > 0) {
+        const listText = result.items.map((item) => `- ${item}`).join("\n");
+        parts.push(`## ${result.section.name}\n\n${listText}`);
+      }
+    } else if (result.type === "children") {
+      if (result.items && Object.keys(result.items).length > 0) {
+        for (const [childName, value] of Object.entries(result.items)) {
+          if (value && String(value).trim()) {
+            parts.push(`## ${childName}\n\n${String(value).trim()}`);
+          }
+        }
+      }
+    } else if (result.type === "children-multiple") {
+      if (result.items && result.items.length > 0) {
+        if (isCommentStyle(result)) {
+          const commentParts = [];
+          for (const item of result.items) {
+            const author =
+              item["Author"] || item["author"] || item["User"] || item["user"] || "";
+            const comment =
+              item["Comment"] || item["Body"] || item["comment"] || item["body"] || "";
+            if (author) {
+              commentParts.push(`**${author}:**\n\n${comment}`);
+            } else if (comment) {
+              commentParts.push(comment);
+            }
+          }
+          if (commentParts.length > 0) {
+            parts.push(
+              `## ${result.section.name}\n\n${commentParts.join("\n\n---\n\n")}`,
+            );
+          }
+        } else {
+          const itemParts = [];
+          for (const item of result.items) {
+            const lines = [];
+            for (const [key, value] of Object.entries(item)) {
+              if (value && String(value).trim()) {
+                lines.push(`    ${key}: ${String(value).trim()}`);
+              }
+            }
+            if (lines.length > 0) itemParts.push(lines.join("\n"));
+          }
+          if (itemParts.length > 0) {
+            parts.push(
+              `## ${result.section.name}\n\n${itemParts.join("\n\n")}`,
+            );
+          }
+        }
       }
     }
+  }
 
-    const totalLength = finalContent.length;
-    let paginatedText = finalContent.substring(
-      startIndex,
-      startIndex + maxLength,
-    );
-    const isTruncated = startIndex + maxLength < totalLength;
+  if (parts.length === 0) {
+    return "(No content extracted from this page.)";
+  }
+
+  const full = parts.join("\n\n---\n\n");
+  const totalLength = full.length;
+  const paginated = full.substring(startIndex, startIndex + maxLength);
+
+  const templateName = template ? template.name : "auto";
+  let metadata = `\n\n---\n[webfetch: template="${templateName}", showing characters ${startIndex} to ${startIndex + paginated.length} of ${totalLength} total.`;
+  if (startIndex + maxLength < totalLength) {
+    metadata += ` Use start_index=${startIndex + maxLength} to read more.`;
+  }
+  metadata += `]`;
+
+  return paginated + metadata;
+}
 
-    let metadata = `\n\n---\n[Document Info: Showing characters ${startIndex} to ${
-      startIndex + paginatedText.length
-    } of ${totalLength} total.`;
+// === COMPOSITION: WEBSEARCH ==============================================
 
-    if (isTruncated) {
-      metadata += ` Use start_index=${startIndex + maxLength} to paginate and read more.`;
+function composeSearchResults(extracted) {
+  // Find the search results section (first children-multiple)
+  const searchSection = extracted.find((r) => r.type === "children-multiple");
+
+  if (!searchSection || !searchSection.items || searchSection.items.length === 0) {
+    // Fall back to section-based output
+    return composeSections(extracted, null, 0, Infinity);
+  }
+
+  const items = searchSection.items;
+  const parts = [];
+
+  for (let i = 0; i < items.length; i++) {
+    const item = items[i];
+    const num = i + 1;
+
+    const title =
+      item["Title"] || item["title"] || Object.values(item)[0] || "";
+    const url =
+      item["URL"] || item["url"] || item["Url"] || "";
+    const snippet =
+      item["Snippet"] || item["snippet"] || "";
+
+    // Filter out non-http URLs and google internal links
+    let cleanUrl = url;
+    if (cleanUrl && !cleanUrl.startsWith("http")) {
+      cleanUrl = ""; // Skip internal/non-web URLs
+    }
+    if (
+      cleanUrl &&
+      (cleanUrl.includes("google.com/search") ||
+        cleanUrl.includes("support.google.com"))
+    ) {
+      cleanUrl = ""; // Skip google internal links
     }
-    metadata += `]`;
 
-    return paginatedText + metadata;
-  } finally {
-    await page.close();
-    await context.close();
+    if (!title) continue;
+
+    const lines = [`[${num}] ${title}`];
+    if (cleanUrl) lines.push(`    URL: ${cleanUrl}`);
+    if (snippet) lines.push(`    Snippet: ${snippet}`);
+
+    parts.push(lines.join("\n"));
   }
+
+  if (parts.length === 0) {
+    return "(No content extracted from this page.)";
+  }
+
+  return `## ${searchSection.section.name}\n\n${parts.join("\n\n")}`;
 }
 
-// ==========================================
-// MCP SERVER INIT & TOOL REGISTRATION
-// ==========================================
+// === GENERIC FALLBACK ====================================================
 
-const server = new McpServer({
-  name: "searchfetch",
-  version: "1.3.0",
-});
+function genericFallback($, startIndex, maxLength) {
+  applyRemove($, null);
+
+  const bodyHtml = $("body").html() || "";
+  let markdown = turndown
+    .turndown(bodyHtml)
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+
+  if (!markdown || markdown.trim().length === 0) {
+    return "(No content extracted from this page.)";
+  }
+
+  const totalLength = markdown.length;
+  const paginated = markdown.substring(startIndex, startIndex + maxLength);
+
+  let metadata = `\n\n---\n[webfetch: template="auto (fallback)", showing characters ${startIndex} to ${startIndex + paginated.length} of ${totalLength} total.`;
+  if (startIndex + maxLength < totalLength) {
+    metadata += ` Use start_index=${startIndex + maxLength} to read more.`;
+  }
+  metadata += `]`;
+
+  return paginated + metadata;
+}
+
+// === SEARCH TEMPLATE RESOLUTION ==========================================
+
+function resolveSearchTemplate(engine, query, region, safeSearch) {
+  const templateName = resolveEngineToTemplateName(engine);
+
+  let template;
+  if (templateName.startsWith("{")) {
+    try {
+      template = JSON.parse(templateName);
+    } catch (e) {
+      throw new Error(`Invalid inline JSON template: ${e.message}`);
+    }
+  } else {
+    template = getTemplateByName(templateName);
+  }
+
+  if (!template.url_template) {
+    throw new Error(
+      `Template '${template.name}' is not a search template (no url_template).`,
+    );
+  }
+
+  const params = mapSearchParams(engine, query, region, safeSearch);
+  let url = resolveUrlTemplate(template, params);
+
+  // Google safe_search: append safe=active to URL
+  if (
+    (engine === "google" || templateName === "google-search") &&
+    safeSearch === true
+  ) {
+    url += "&safe=active";
+  }
 
-server.tool(
+  return { template, url };
+}
+
+// === MCP SERVER & TOOLS ==================================================
+
+const server = new McpServer({ name: "searchfetch", version: "3.0.0" });
+
+// --- websearch tool ---
+
+server.registerTool(
   "websearch",
-  "Search the web using DuckDuckGo or Google. Returns a clean list of titles, URLs, and snippets. Excellent for researching general knowledge, news, and finding URLs.",
   {
-    query: z.string().describe("The search query string."),
-    engine: z
-      .enum(["duckduckgo", "google"])
-      .default("duckduckgo")
-      .describe("Search engine to use (default: duckduckgo)."),
-    max_results: z
-      .number()
-      .default(10)
-      .describe("Maximum number of results to return (default: 10)."),
-    region: z
-      .string()
-      .default("wt-wt")
-      .describe("Region code (e.g., 'us-en'). Only applies to DuckDuckGo."),
-    safe_search: z
-      .string()
-      .default("-1")
-      .describe(
-        "'-1' for Moderate, '1' for Strict, '-2' for Off. Only applies to DuckDuckGo.",
-      ),
+    title: "Web Search",
+    description:
+      "Search the web using DuckDuckGo or Google. Returns a clean list of titles, URLs, and snippets. Excellent for researching general knowledge, news, and finding URLs.",
+    inputSchema: z.object({
+      query: z.string().describe("The search query string."),
+      engine: z
+        .string()
+        .default("duckduckgo")
+        .describe(
+          "Search engine to use. Can be 'duckduckgo' or 'google'. Default is 'duckduckgo'.",
+        ),
+      region: z
+        .string()
+        .nullable()
+        .default(null)
+        .describe(
+          "Region and language code to localize search results (e.g., 'us-en', 'uk-en', 'de-de'). For DuckDuckGo it maps directly. For Google, 'us' is country code and 'en' is language. Default is null (uses template default).",
+        ),
+      safe_search: z
+        .boolean()
+        .nullable()
+        .default(null)
+        .describe(
+          "Enable safe search filtering. null = use template default. Applies to both DuckDuckGo and Google.",
+        ),
+      max_results: z
+        .number()
+        .default(10)
+        .describe("Maximum number of search results to return. Default is 10."),
+      block_media: z
+        .boolean()
+        .default(true)
+        .describe(
+          "Block images, videos, and fonts entirely at the network layer. Default is true.",
+        ),
+    }),
   },
-  async ({ query, engine, max_results, region, safe_search }) => {
+  async ({ query, engine, region, safe_search, max_results, block_media }) => {
     try {
-      const result = await executeSearch(
+      // 1. Resolve search template (+ url_params mapping + url building)
+      const { template, url } = resolveSearchTemplate(
+        engine,
         query,
-        max_results,
         region,
         safe_search,
-        engine,
       );
+
+      // 2. Fetch
+      const html = await fetchHtmlWithRetry(url, template, block_media);
+
+      // 3. Extract
+      const $ = cheerio.load(html);
+      applyRemove($, template);
+
+      const pageOrigin = new URL(url).origin;
+      const context = {
+        origin: pageOrigin,
+        isWebsearch: true,
+        maxResultsOverride: max_results,
+        _maxResultsConsumed: false,
+      };
+
+      const extracted = extractTemplate($, template, context);
+
+      // 4. Compose
+      const result = composeSearchResults(extracted);
+
       return { content: [{ type: "text", text: result }] };
-    } catch (error) {
-      logger.error("Search Tool failed:", error);
+    } catch (err) {
       return {
-        content: [{ type: "text", text: `Search Error: ${error.message}` }],
+        content: [{ type: "text", text: `Search Error: ${err.message}` }],
         isError: true,
       };
     }
   },
 );
 
-server.tool(
+// --- webfetch tool ---
+
+server.registerTool(
   "webfetch",
-  "Fetch and extract the main text content from any webpage. Fully executes JavaScript to load React/SPAs and aggressively strips images/media (including base64) to save context tokens.",
   {
-    url: z
-      .string()
-      .url()
-      .describe(
+    title: "Web Fetch",
+    description:
+      "Fetch and extract the main text content from any webpage. Fully executes JavaScript to load React/SPAs and aggressively strips images/media (including base64) to save context tokens.",
+    inputSchema: z.object({
+      url: z.string().describe(
         "The full URL of the webpage to fetch (must start with http/https).",
       ),
-    format: z
-      .enum(["markdown", "clean_html", "raw_html"])
-      .default("markdown")
-      .describe(
-        "Output format. Markdown is highly recommended to save context tokens.",
-      ),
-    start_index: z
-      .number()
-      .default(0)
-      .describe("Character offset to start reading from for pagination."),
-    max_length: z
-      .number()
-      .default(10000)
-      .describe("Maximum characters to return per request (default: 10000)."),
-    block_media: z
-      .boolean()
-      .default(true)
-      .describe(
-        "Block images/videos/fonts to drastically speed up rendering (default: true).",
-      ),
+      template: z
+        .string()
+        .default("auto")
+        .describe(
+          "Template to use: 'auto' (auto-detect from URL), a built-in name, or inline JSON.",
+        ),
+      start_index: z
+        .number()
+        .default(0)
+        .describe(
+          "Character offset for pagination. Default: 0.",
+        ),
+      max_length: z
+        .number()
+        .default(10000)
+        .describe(
+          "Maximum characters to return per request. Default is 10000.",
+        ),
+      block_media: z
+        .boolean()
+        .default(true)
+        .describe(
+          "Block images, videos, and fonts entirely at the network layer. Default is true.",
+        ),
+    }),
   },
-  async ({ url, format, start_index, max_length, block_media }) => {
+  async ({ url, template: templateParam, start_index, max_length, block_media }) => {
     try {
-      const result = await executeFetch(
-        url,
-        format,
-        start_index,
-        max_length,
-        block_media,
-      );
-      return { content: [{ type: "text", text: result }] };
-    } catch (error) {
-      logger.error(`Fetch Tool failed on ${url}:`, error);
+      // 1. Resolve template
+      let template;
+
+      if (templateParam.startsWith("{")) {
+        try {
+          template = JSON.parse(templateParam);
+        } catch (e) {
+          throw new Error(`Invalid inline JSON template: ${e.message}`);
+        }
+      } else if (templateParam === "auto") {
+        template = detectTemplateByUrl(url);
+      } else {
+        template = getTemplateByName(templateParam);
+      }
+
+      // 2. Fetch
+      const html = await fetchHtmlWithRetry(url, template, block_media);
+
+      // 3. Extract and compose
+      const $ = cheerio.load(html);
+
+      if (template) {
+        applyRemove($, template);
+
+        const pageOrigin = new URL(url).origin;
+        const context = {
+          origin: pageOrigin,
+          isWebsearch: false,
+        };
+
+        const extracted = extractTemplate($, template, context);
+        const result = composeSections(extracted, template, start_index, max_length);
+        return { content: [{ type: "text", text: result }] };
+      } else {
+        // Generic fallback
+        const result = genericFallback($, start_index, max_length);
+        return { content: [{ type: "text", text: result }] };
+      }
+    } catch (err) {
       return {
-        content: [{ type: "text", text: `Fetch Error: ${error.message}` }],
+        content: [{ type: "text", text: `Fetch Error: ${err.message}` }],
         isError: true,
       };
     }
   },
 );
 
-// ==========================================
-// BOOTSTRAP
-// ==========================================
+// === MAIN =================================================================
 
 async function main() {
-  logger.info("Initializing MCP Server...");
-
   await ensureBinary();
-
-  // Re-enable STDOUT right before protocol hook-in
   process.stdout.write = originalStdoutWrite;
-
   const transport = new StdioServerTransport();
   await server.connect(transport);
-
-  logger.info("searchfetch successfully connected and listening for requests.");
 }
 
 main().catch((err) => {
-  logger.error("Fatal error during startup:", err);
   process.exit(1);
 });