peeky-search 1.0.10

package/dist/chunk-S3WZDJCP.js

@@ -0,0 +1,2716 @@
+// src/preprocessing/tokenize.ts
+import { stemmer } from "stemmer";
+var STOP_WORDS = /* @__PURE__ */ new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "by",
+  "for",
+  "from",
+  "has",
+  "he",
+  "in",
+  "is",
+  "it",
+  "its",
+  "of",
+  "on",
+  "that",
+  "the",
+  "to",
+  "was",
+  "were",
+  "will",
+  "with",
+  "the",
+  "this",
+  "but",
+  "they",
+  "have",
+  "had",
+  "what",
+  "when",
+  "where",
+  "who",
+  "which",
+  "why",
+  "how",
+  "all",
+  "each",
+  "every",
+  "both",
+  "few",
+  "more",
+  "most",
+  "other",
+  "some",
+  "such",
+  "no",
+  "nor",
+  "not",
+  "only",
+  "own",
+  "same",
+  "so",
+  "than",
+  "too",
+  "very",
+  "can",
+  "just",
+  "should",
+  "now",
+  "or",
+  "if",
+  "then",
+  "else",
+  "been",
+  "being",
+  "do",
+  "does",
+  "did",
+  "doing",
+  "would",
+  "could",
+  "might",
+  "must",
+  "shall",
+  "may",
+  "about",
+  "above",
+  "after",
+  "again",
+  "against",
+  "below",
+  "between",
+  "into",
+  "through",
+  "during",
+  "before",
+  "under",
+  "over",
+  "out",
+  "up",
+  "down",
+  "off",
+  "once",
+  "here",
+  "there",
+  "any",
+  "your",
+  "you",
+  "we",
+  "our",
+  "us",
+  "i",
+  "me",
+  "my",
+  "myself",
+  "him",
+  "her",
+  "them",
+  "their",
+  "his",
+  "she",
+  "he",
+  "it",
+  "its",
+  "itself"
+]);
+function splitCamelCase(text) {
+  return text.replace(/([a-z])([A-Z])/g, "$1 $2").replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
+}
+function normalizeText(text) {
+  return splitCamelCase(text).toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").replace(/\s+/g, " ").trim();
+}
+function tokenize(text, options = {}) {
+  const {
+    removeStopWords = true,
+    applyStemming = true,
+    minLength = 2
+  } = options;
+  const normalized = normalizeText(text);
+  let tokens = normalized.split(/\s+/).filter((t) => t.length >= minLength);
+  if (removeStopWords) {
+    tokens = tokens.filter((t) => !STOP_WORDS.has(t));
+  }
+  if (applyStemming) {
+    tokens = tokens.map(stemmer);
+  }
+  return tokens;
+}
+function buildTermFrequencyMap(tokens) {
+  const tf = {};
+  for (const token of tokens) {
+    tf[token] = (tf[token] ?? 0) + 1;
+  }
+  return tf;
+}
+function jaccardSimilarity(tokensA, tokensB) {
+  const setA = new Set(tokensA);
+  const setB = new Set(tokensB);
+  let intersection = 0;
+  for (const token of setA) {
+    if (setB.has(token)) {
+      intersection++;
+    }
+  }
+  const union = setA.size + setB.size - intersection;
+  if (union === 0) return 0;
+  return intersection / union;
+}
+function termOverlapRatio(tokensA, tokensB) {
+  if (tokensA.length === 0) return 0;
+  const setB = new Set(tokensB);
+  let overlap = 0;
+  for (const token of tokensA) {
+    if (setB.has(token)) {
+      overlap++;
+    }
+  }
+  return overlap / tokensA.length;
+}
+
+// src/output/excerpts.ts
+var DEFAULT_CONFIG = {
+  maxExcerpts: 3,
+  charBudget: 2e3,
+  minExcerptChars: 50
+};
+function chunkToExcerpt(chunk) {
+  return {
+    text: chunk.text,
+    headingPath: chunk.headingPath,
+    score: chunk.score,
+    charCount: chunk.charCount
+  };
+}
+function assembleExcerpts(chunks, query, config = {}) {
+  const {
+    maxExcerpts = DEFAULT_CONFIG.maxExcerpts,
+    charBudget = DEFAULT_CONFIG.charBudget,
+    minExcerptChars = DEFAULT_CONFIG.minExcerptChars
+  } = config;
+  const sorted = [...chunks].sort((a, b) => {
+    const d = b.score - a.score;
+    if (d !== 0) return d;
+    return a.anchorIndex - b.anchorIndex;
+  });
+  const excerpts = [];
+  let totalChars = 0;
+  for (const chunk of sorted) {
+    if (chunk.charCount < minExcerptChars) {
+      continue;
+    }
+    if (totalChars + chunk.charCount > charBudget) {
+      continue;
+    }
+    if (excerpts.length >= maxExcerpts) {
+      break;
+    }
+    excerpts.push(chunkToExcerpt(chunk));
+    totalChars += chunk.charCount;
+  }
+  return {
+    excerpts,
+    totalChars,
+    query
+  };
+}
+function formatExcerpts(result) {
+  const lines = [];
+  lines.push(`Query: "${result.query}"`);
+  lines.push(`Total characters: ${result.totalChars}`);
+  lines.push("");
+  for (let i = 0; i < result.excerpts.length; i++) {
+    const excerpt = result.excerpts[i];
+    if (excerpt === void 0) continue;
+    lines.push(`--- Excerpt ${i + 1} (score: ${excerpt.score.toFixed(3)}, ${excerpt.charCount} chars) ---`);
+    if (excerpt.headingPath.length > 0) {
+      lines.push(`[${excerpt.headingPath.join(" > ")}]`);
+    }
+    lines.push(excerpt.text);
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+
+// src/utils/logger.ts
+var Logger = class _Logger {
+  static instance;
+  timings = [];
+  timingEnabled = false;
+  constructor() {
+  }
+  static getInstance() {
+    if (!_Logger.instance) {
+      _Logger.instance = new _Logger();
+    }
+    return _Logger.instance;
+  }
+  log(message) {
+    console.log(`[${(/* @__PURE__ */ new Date()).toISOString()}] [INFO] ${message}`);
+  }
+  error(message) {
+    console.error(`[${(/* @__PURE__ */ new Date()).toISOString()}] [ERROR] ${message}`);
+  }
+  debug(message, enabled = true) {
+    if (enabled) {
+      console.error(`[${(/* @__PURE__ */ new Date()).toISOString()}] [DEBUG] ${message}`);
+    }
+  }
+  /**
+   * Enable or disable timing collection
+   */
+  setTimingEnabled(enabled) {
+    this.timingEnabled = enabled;
+    if (enabled) {
+      this.timings = [];
+    }
+  }
+  /**
+   * Time a synchronous function and record the result
+   */
+  time(label, fn) {
+    if (!this.timingEnabled) {
+      return fn();
+    }
+    const start = performance.now();
+    const result = fn();
+    const durationMs = performance.now() - start;
+    this.timings.push({ label, durationMs });
+    return result;
+  }
+  /**
+   * Time an async function and record the result
+   */
+  async timeAsync(label, fn) {
+    if (!this.timingEnabled) {
+      return fn();
+    }
+    const start = performance.now();
+    const result = await fn();
+    const durationMs = performance.now() - start;
+    this.timings.push({ label, durationMs });
+    return result;
+  }
+  /**
+   * Get all recorded timings
+   */
+  getTimings() {
+    return [...this.timings];
+  }
+  /**
+   * Clear recorded timings
+   */
+  clearTimings() {
+    this.timings = [];
+  }
+  /**
+   * Record a timing directly (useful for manual timing measurements)
+   */
+  recordTiming(label, durationMs) {
+    if (this.timingEnabled) {
+      this.timings.push({ label, durationMs });
+    }
+  }
+  /**
+   * Print timing summary to console
+   */
+  printTimings() {
+    if (this.timings.length === 0) {
+      console.error("[TIMING] No timings recorded");
+      return;
+    }
+    console.error("\n[TIMING] === Performance Summary ===");
+    const total = this.timings.reduce((sum, t) => sum + t.durationMs, 0);
+    for (const timing of this.timings) {
+      const pct = (timing.durationMs / total * 100).toFixed(1);
+      console.error(`[TIMING] ${timing.label.padEnd(30)} ${timing.durationMs.toFixed(2).padStart(8)}ms (${pct.padStart(5)}%)`);
+    }
+    console.error(`[TIMING] ${"TOTAL".padEnd(30)} ${total.toFixed(2).padStart(8)}ms`);
+    console.error("[TIMING] ================================\n");
+  }
+};
+var logger_default = Logger;
+
+// src/preprocessing/strip.ts
+import * as cheerio from "cheerio";
+function isElement(node) {
+  return node.type === "tag";
+}
+var REMOVE_ELEMENTS = [
+  "script",
+  "style",
+  "link",
+  "img",
+  "iframe",
+  "video",
+  "audio",
+  "object",
+  "embed",
+  "param",
+  "applet",
+  "noscript",
+  "svg",
+  "canvas",
+  "map",
+  "area",
+  // Interactive elements that are never content
+  "button",
+  "input",
+  "select",
+  "textarea",
+  "form",
+  "label",
+  "fieldset",
+  "legend",
+  "datalist",
+  "output",
+  "progress",
+  "meter"
+];
+var BOILERPLATE_ELEMENTS = [
+  "nav",
+  "footer",
+  "aside",
+  "header"
+];
+var BOILERPLATE_PATTERNS = [
+  /nav(igation)?/i,
+  /footer/i,
+  /header/i,
+  /sidebar/i,
+  /menu/i,
+  /breadcrumb/i,
+  /cookie/i,
+  /consent/i,
+  /banner/i,
+  /advert(isement)?/i,
+  /ads?[-_]?/i,
+  /social/i,
+  /share/i,
+  /comment/i,
+  /related/i,
+  /recommend/i,
+  /popup/i,
+  /modal/i,
+  /newsletter/i,
+  /subscribe/i,
+  /signup/i,
+  /login/i,
+  /signin/i,
+  /search/i,
+  /widget/i,
+  /toolbar/i,
+  /promo/i
+];
+var UI_ELEMENT_PATTERNS = [
+  /copy[-_]?(button|link|code|markdown)?/i,
+  /share[-_]?(button|link|menu)?/i,
+  /action[-_]?(bar|menu|buttons?)/i,
+  /btn[-_]?(copy|share|action)/i,
+  /clipboard/i,
+  /open[-_]?in/i,
+  /chat[-_]?(gpt|button)/i,
+  /ai[-_]?(chat|assistant)/i,
+  /feedback/i,
+  /edit[-_]?(page|this|on[-_]?github)/i,
+  /page[-_]?actions?/i,
+  /toc|table[-_]?of[-_]?contents/i,
+  /on[-_]?this[-_]?page/i,
+  /sticky[-_]?(nav|sidebar|toc)/i,
+  /floating[-_]?(menu|button|action)/i
+];
+var UI_TEXT_PATTERNS = [
+  /^copy\s*(as\s*)?(markdown|code|text)?$/i,
+  /^open\s+in\s+\w+/i,
+  // "Open in X" for any service
+  /^(share|copy)\s*(this|link|page)?$/i,
+  /^edit\s*(this\s*)?(page|on\s*github)?$/i,
+  /^(give\s*)?feedback$/i,
+  /^(scroll\s*to\s*)?top$/i,
+  /^on\s+this\s+page$/i,
+  /^table\s+of\s+contents$/i
+];
+function isBoilerplateElement(el) {
+  if (el.type !== "tag") return false;
+  const id = el.attribs?.id ?? "";
+  const className = el.attribs?.class ?? "";
+  const combined = `${id} ${className}`;
+  return BOILERPLATE_PATTERNS.some((pattern) => pattern.test(combined));
+}
+function isUIElement(el) {
+  if (el.type !== "tag") return false;
+  const id = el.attribs?.id ?? "";
+  const className = el.attribs?.class ?? "";
+  const combined = `${id} ${className}`;
+  return UI_ELEMENT_PATTERNS.some((pattern) => pattern.test(combined));
+}
+function isUIText(text) {
+  const normalized = text.replace(/\s+/g, " ").trim();
+  return UI_TEXT_PATTERNS.some((pattern) => pattern.test(normalized));
+}
+function stripHtml(html) {
+  const $ = cheerio.load(html);
+  for (const selector of REMOVE_ELEMENTS) {
+    $(selector).remove();
+  }
+  return $;
+}
+function removeBoilerplate($) {
+  const mainContent = $("main, article, [role='main']");
+  mainContent.attr("data-peeky-protect", "true");
+  mainContent.find("*").attr("data-peeky-protect", "true");
+  for (const selector of BOILERPLATE_ELEMENTS) {
+    $(selector).each((_, el) => {
+      const $el = $(el);
+      if (!$el.attr("data-peeky-protect") && !$el.find("[data-peeky-protect]").length) {
+        $el.remove();
+      }
+    });
+  }
+  $("*").each((_, el) => {
+    if (!isElement(el)) return;
+    const $el = $(el);
+    if ($el.attr("data-peeky-protect")) return;
+    if ($el.find("[data-peeky-protect]").length) return;
+    if (isBoilerplateElement(el)) {
+      $el.remove();
+    }
+  });
+  $("[data-peeky-protect]").removeAttr("data-peeky-protect");
+}
+function removeUIElements($, container) {
+  container.find("*").each((_, el) => {
+    if (!isElement(el)) return;
+    if (isUIElement(el)) {
+      $(el).remove();
+    }
+  });
+  container.find("button, a, span, div").each((_, el) => {
+    const $el = $(el);
+    const text = $el.text();
+    if (text.length < 50 && isUIText(text)) {
+      $el.remove();
+    }
+  });
+  const uiSelectors = [
+    "[data-copy]",
+    "[data-clipboard]",
+    "[aria-label*='copy']",
+    "[aria-label*='share']",
+    "[title*='Copy']",
+    "[title*='Share']",
+    ".copy-button",
+    ".share-button",
+    "[class*='ActionMenu']",
+    "[class*='PageActions']",
+    // Tab navigation and anchor links within content
+    "[role='tablist']",
+    "[role='tab']",
+    ".tabs",
+    ".tab-list",
+    "[class*='TabList']",
+    "[class*='Tabs']",
+    // Anchor/jump links
+    "[class*='anchor-link']",
+    "[class*='heading-link']",
+    "a[href^='#']:empty",
+    // Empty anchor links
+    // Sticky sidebars (often contain TOC and share links)
+    "[class*='sticky']"
+  ];
+  for (const selector of uiSelectors) {
+    container.find(selector).remove();
+  }
+  container.find("p, div, span").each((_, el) => {
+    const $el = $(el);
+    const links = $el.find("a");
+    const text = $el.text().replace(/\s+/g, " ").trim();
+    if (links.length >= 3) {
+      let linkTextLen = 0;
+      links.each((_2, link) => {
+        linkTextLen += $(link).text().length;
+      });
+      if (linkTextLen > text.length * 0.8 && text.length < 200) {
+        $el.remove();
+      }
+    }
+  });
+}
+function findMainContent($) {
+  const mainEl = $("main");
+  if (mainEl.length > 0) {
+    return {
+      element: mainEl,
+      selector: "main",
+      score: 100
+    };
+  }
+  const articleEl = $("article");
+  if (articleEl.length > 0) {
+    return {
+      element: articleEl,
+      selector: "article",
+      score: 90
+    };
+  }
+  const candidates = [];
+  $("body").children().each((_, elem) => {
+    if (elem.type !== "tag") return;
+    const $el = $(elem);
+    if (BOILERPLATE_ELEMENTS.includes(elem.tagName.toLowerCase())) return;
+    if (isBoilerplateElement(elem)) return;
+    const textLen = $el.text().replace(/\s+/g, " ").trim().length;
+    let linkTextLen = 0;
+    $el.find("a").each((_2, link) => {
+      linkTextLen += $(link).text().replace(/\s+/g, " ").trim().length;
+    });
+    const score = textLen - 2 * linkTextLen;
+    let selector = elem.tagName;
+    if (elem.attribs?.id) {
+      selector += `#${elem.attribs.id}`;
+    }
+    if (elem.attribs?.class) {
+      selector += `.${elem.attribs.class.split(" ").join(".")}`;
+    }
+    candidates.push({ element: $el, selector, score });
+  });
+  const contentSelectors = [
+    "[role='main']",
+    "#content",
+    "#main-content",
+    ".content",
+    ".main-content",
+    ".post-content",
+    ".article-content",
+    ".entry-content"
+  ];
+  for (const sel of contentSelectors) {
+    const $el = $(sel);
+    if ($el.length > 0) {
+      const textLen = $el.text().replace(/\s+/g, " ").trim().length;
+      let linkTextLen = 0;
+      $el.find("a").each((_, link) => {
+        linkTextLen += $(link).text().replace(/\s+/g, " ").trim().length;
+      });
+      const score = textLen - 2 * linkTextLen;
+      candidates.push({ element: $el, selector: sel, score });
+    }
+  }
+  if (candidates.length === 0) {
+    return null;
+  }
+  candidates.sort((a, b) => {
+    const d = b.score - a.score;
+    if (d !== 0) return d;
+    return a.selector.localeCompare(b.selector);
+  });
+  const best = candidates[0];
+  return best ?? null;
+}
+function preprocessHtml(html) {
+  const $ = stripHtml(html);
+  removeBoilerplate($);
+  const candidate = findMainContent($);
+  if (candidate === null) {
+    return { $, mainContent: null, selector: null };
+  }
+  removeUIElements($, candidate.element);
+  return {
+    $,
+    mainContent: candidate.element,
+    selector: candidate.selector
+  };
+}
+
+// src/preprocessing/segment.ts
+function isBlockTag(tagName) {
+  return /^h[1-6]$/.test(tagName) || tagName === "p" || tagName === "li" || tagName === "pre";
+}
+function extractCodeText($, $pre) {
+  const $lines = $pre.find(".line, .code-line, [class*='line']");
+  if ($lines.length > 0) {
+    const lines = [];
+    $lines.each((_, el) => {
+      const lineText = $(el).text().trim();
+      if (lineText.length > 0) {
+        lines.push(lineText);
+      }
+    });
+    return lines.join("\n").trim();
+  }
+  const html = $pre.html() ?? "";
+  if (html.includes("<br")) {
+    const withNewlines = html.replace(/<br\s*\/?>/gi, "\n");
+    const $temp = $.load(`<div>${withNewlines}</div>`);
+    return $temp("div").text().trim();
+  }
+  const rawText = $pre.text() ?? "";
+  return rawText.trim();
+}
+function walk($, $node, path, blocks, indexRef, skipNav) {
+  if (skipNav && $node.is("nav")) return;
+  const rawTag = $node.prop("tagName");
+  const tagName = typeof rawTag === "string" ? rawTag.toLowerCase() : "";
+  if (tagName !== "" && isBlockTag(tagName)) {
+    let text;
+    if (tagName === "pre") {
+      text = extractCodeText($, $node);
+    } else {
+      const rawText = $node.text() ?? "";
+      text = rawText.replace(/\s+/g, " ").trim();
+    }
+    if (text.length === 0) return;
+    blocks.push({
+      type: tagName,
+      text,
+      index: indexRef.current++,
+      headingPath: [...path]
+    });
+    if (/^h[1-6]$/.test(tagName)) {
+      const levelChar = tagName[1];
+      const level = levelChar !== void 0 ? parseInt(levelChar, 10) : 1;
+      const targetLength = level - 1;
+      if (targetLength < path.length) {
+        path.length = targetLength;
+      }
+      path.push(text);
+    }
+    return;
+  }
+  $node.children().each((_, child) => {
+    if (child.type === "tag") {
+      walk($, $(child), path, blocks, indexRef, skipNav);
+    }
+  });
+}
+function extractBlocks($, container, options = {}) {
+  const { skipNav = true } = options;
+  const blocks = [];
+  const path = [];
+  const indexRef = { current: 0 };
+  walk($, container, path, blocks, indexRef, skipNav);
+  return blocks;
+}
+var ABBREVIATIONS = /* @__PURE__ */ new Set([
+  "mr",
+  "mrs",
+  "ms",
+  "dr",
+  "prof",
+  "sr",
+  "jr",
+  "vs",
+  "etc",
+  "inc",
+  "ltd",
+  "st",
+  "ave",
+  "blvd",
+  "rd",
+  "e.g",
+  "i.e",
+  "cf",
+  "al",
+  "fig",
+  "vol",
+  "no"
+]);
+var CODE_BLOCK_CLEANUP_PATTERNS = [
+  /Try$/,
+  // TypeScript playground "Try" suffix
+  /Run$/,
+  // "Run" button text
+  /Copy$/,
+  // Copy button text
+  /\s*Try\s*$/,
+  // With whitespace
+  /\s*Run\s*$/,
+  /\s*Copy\s*$/,
+  /Open in Playground$/i,
+  // Playground links
+  /Open in CodeSandbox$/i,
+  /Open in StackBlitz$/i,
+  /Edit on GitHub$/i,
+  /View on GitHub$/i
+];
+function cleanCodeBlock(text) {
+  let cleaned = text;
+  for (const pattern of CODE_BLOCK_CLEANUP_PATTERNS) {
+    cleaned = cleaned.replace(pattern, "");
+  }
+  return cleaned.trim();
+}
+function splitIntoSentences(text) {
+  if (text.length === 0) return [];
+  const normalized = text.replace(/\s+/g, " ").trim();
+  const sentences = [];
+  let current = "";
+  let i = 0;
+  while (i < normalized.length) {
+    const char = normalized[i];
+    current += char;
+    if (char === "." || char === "!" || char === "?") {
+      const nextChar = normalized[i + 1];
+      const afterNext = normalized[i + 2];
+      if (nextChar === void 0) {
+        const trimmed2 = current.trim();
+        if (trimmed2.length > 0) {
+          sentences.push(trimmed2);
+        }
+        current = "";
+      } else if (nextChar === " " && afterNext !== void 0 && /[A-Z]/.test(afterNext)) {
+        const wordMatch = current.match(/(\w+)\.$/);
+        const word = wordMatch?.[1]?.toLowerCase() ?? "";
+        if (!ABBREVIATIONS.has(word)) {
+          const trimmed2 = current.trim();
+          if (trimmed2.length > 0) {
+            sentences.push(trimmed2);
+          }
+          current = "";
+          i++;
+        }
+      }
+    }
+    i++;
+  }
+  const trimmed = current.trim();
+  if (trimmed.length > 0) {
+    sentences.push(trimmed);
+  }
+  return sentences;
+}
+function segmentBlocks(blocks, options = {}) {
+  const { tokenizeOptions } = options;
+  const sentences = [];
+  let globalIndex = 0;
+  const totalBlocks = blocks.length;
+  for (const block of blocks) {
+    const isHeading = /^h[1-6]$/.test(block.type);
+    const isCode = block.type === "pre";
+    const blockText = isCode ? cleanCodeBlock(block.text) : block.text;
+    const blockSentences = isHeading || isCode ? [blockText] : splitIntoSentences(blockText);
+    for (let sentenceIndex = 0; sentenceIndex < blockSentences.length; sentenceIndex++) {
+      const text = blockSentences[sentenceIndex];
+      if (text === void 0 || text.length === 0) continue;
+      const tokens = tokenize(text, tokenizeOptions);
+      sentences.push({
+        text,
+        tokens,
+        blockIndex: block.index,
+        sentenceIndex,
+        globalIndex,
+        headingPath: block.headingPath,
+        position: totalBlocks > 1 ? block.index / (totalBlocks - 1) : 0,
+        blockType: block.type
+      });
+      globalIndex++;
+    }
+  }
+  return sentences;
+}
+function segmentHtml($, container, options = {}) {
+  const { skipNav = true, tokenizeOptions } = options;
+  const blocks = extractBlocks($, container, { skipNav });
+  const sentences = segmentBlocks(blocks, { tokenizeOptions });
+  return { blocks, sentences };
+}
+
+// src/scoring/bm25.ts
+var DEFAULT_K1 = 1.2;
+var DEFAULT_B = 0.5;
+function computeDocumentStats(sentences) {
+  const docFrequency = {};
+  let totalLength = 0;
+  for (const sentence of sentences) {
+    const uniqueTerms = new Set(sentence.tokens);
+    for (const term of uniqueTerms) {
+      docFrequency[term] = (docFrequency[term] ?? 0) + 1;
+    }
+    totalLength += sentence.tokens.length;
+  }
+  return {
+    totalDocs: sentences.length,
+    avgDocLength: sentences.length > 0 ? totalLength / sentences.length : 0,
+    docFrequency
+  };
+}
+function calculateIdf(term, stats) {
+  const df = stats.docFrequency[term] ?? 0;
+  const N = stats.totalDocs;
+  if (df === 0) {
+    return Math.log((N + 0.5) / 0.5 + 1);
+  }
+  return Math.log((N - df + 0.5) / (df + 0.5) + 1);
+}
+function createBM25Scorer(sentences, config = {}) {
+  const { k1 = DEFAULT_K1, b = DEFAULT_B } = config;
+  const stats = computeDocumentStats(sentences);
+  function getIdf(term) {
+    return calculateIdf(term, stats);
+  }
+  function scoreWithTf(queryTokens, docTf, docLength) {
+    let score2 = 0;
+    for (const term of queryTokens) {
+      const tf = docTf[term] ?? 0;
+      if (tf === 0) continue;
+      const idf = getIdf(term);
+      const numerator = tf * (k1 + 1);
+      const denominator = tf + k1 * (1 - b + b * (docLength / stats.avgDocLength));
+      const termScore = idf * (numerator / denominator);
+      score2 += termScore;
+    }
+    return score2;
+  }
+  function score(queryTokens, docTokens) {
+    const docTf = buildTermFrequencyMap(docTokens);
+    return scoreWithTf(queryTokens, docTf, docTokens.length);
+  }
+  return {
+    score,
+    scoreWithTf,
+    getIdf,
+    stats
+  };
+}
+
+// src/scoring/heuristics.ts
+var DEFAULT_WEIGHTS = {
+  position: 0.05,
+  headingProximity: 0.11,
+  density: 0.09,
+  structure: 0.11,
+  proximity: 0.14,
+  headingPath: 0.17,
+  coverage: 0.16,
+  outlier: 0.09,
+  metaSection: 0.08
+};
+var META_SECTION_PATTERNS = [
+  // Introduction/overview patterns
+  /^introduction$/i,
+  /^overview$/i,
+  /^about(\s+this)?(\s+(article|guide|tutorial|post|page))?$/i,
+  /^what('s|\s+is)\s+this/i,
+  /^what\s+you('ll|.will)\s+(learn|cover|build|create)/i,
+  /^what\s+we('ll|.will)\s+(learn|cover|build|create)/i,
+  /^in\s+this\s+(article|guide|tutorial|post)/i,
+  /^getting\s+started$/i,
+  /^before\s+(you\s+)?begin/i,
+  /^prerequisites?$/i,
+  /^requirements?$/i,
+  /^background$/i,
+  /^context$/i,
+  // Summary/conclusion patterns
+  /^summary$/i,
+  /^conclusion$/i,
+  /^tl;?\s*dr$/i,
+  /^takeaways?$/i,
+  /^key\s+takeaways?$/i,
+  /^key\s+points?$/i,
+  /^wrapping\s+up$/i,
+  /^final\s+thoughts?$/i,
+  /^closing\s+thoughts?$/i,
+  /^in\s+summary$/i,
+  /^to\s+summarize$/i,
+  /^recap$/i,
+  // Next steps/related content
+  /^next\s+steps?$/i,
+  /^what('s|.is)\s+next/i,
+  /^further\s+reading$/i,
+  /^additional\s+resources?$/i,
+  /^related\s+(articles?|posts?|content|links?|resources?)/i,
+  /^see\s+also$/i,
+  /^learn\s+more$/i,
+  /^more\s+resources?$/i,
+  /^references?$/i,
+  /^sources?$/i,
+  /^bibliography$/i,
+  /^credits?$/i,
+  /^acknowledgements?$/i,
+  // Author/meta content
+  /^about\s+(the\s+)?author/i,
+  /^author(\s+bio)?$/i,
+  /^bio(graphy)?$/i,
+  /^written\s+by$/i,
+  /^posted\s+by$/i,
+  /^published\s+by$/i,
+  // Engagement/social
+  /^comments?$/i,
+  /^feedback$/i,
+  /^discussion$/i,
+  /^share(\s+this)?$/i,
+  /^subscribe$/i,
+  /^newsletter$/i,
+  /^follow(\s+us)?$/i,
+  /^connect(\s+with\s+us)?$/i,
+  /^join(\s+us)?$/i,
+  /^support(\s+us)?$/i,
+  /^donate$/i,
+  /^buy\s+me\s+a\s+coffee/i,
+  /^sponsor/i,
+  // Navigation/structural
+  /^table\s+of\s+contents?$/i,
+  /^contents?$/i,
+  /^toc$/i,
+  /^navigation$/i,
+  /^menu$/i,
+  /^sidebar$/i,
+  /^footer$/i,
+  /^header$/i,
+  /^breadcrumb/i,
+  /^skip\s+to/i,
+  // Cross-reference sections (often just link lists)
+  /^see\s+also$/i,
+  /^more\s+info(rmation)?$/i,
+  /^external\s+links?$/i,
+  /^useful\s+links?$/i,
+  /^quick\s+links?$/i,
+  /^related\s+topics?$/i,
+  /^related\s+guides?$/i,
+  /^related\s+tutorials?$/i,
+  /^related\s+documentation$/i,
+  /^other\s+resources?$/i,
+  /^specifications?$/i,
+  /^browser\s+compatibility$/i,
+  /^browser\s+support$/i,
+  // Disclaimers/legal
+  /^disclaimer$/i,
+  /^disclosure$/i,
+  /^affiliate\s+disclosure/i,
+  /^privacy(\s+policy)?$/i,
+  /^terms(\s+(of\s+)?(use|service))?$/i,
+  /^copyright$/i,
+  /^legal$/i,
+  // Promotional
+  /^featured$/i,
+  /^trending$/i,
+  /^popular(\s+posts?)?$/i,
+  /^recommended$/i,
+  /^you\s+might\s+(also\s+)?like$/i,
+  /^related\s+posts?$/i,
+  /^top\s+stories$/i,
+  /^latest(\s+posts?)?$/i,
+  /^recent(\s+posts?)?$/i,
+  /^archives?$/i,
+  /^categories$/i,
+  /^tags?$/i
+];
+function calculatePositionScore(sentence) {
+  const position = sentence.position;
+  if (position <= 0.3) {
+    return 1 - position / 0.3 * 0.3;
+  } else if (position <= 0.7) {
+    return 0.7 - (position - 0.3) / 0.4 * 0.2;
+  } else {
+    return 0.5 - (position - 0.7) / 0.3 * 0.2;
+  }
+}
+function calculateHeadingProximityScore(sentence, queryTokens, allSentences) {
+  let nearestHeading = null;
+  let distance = Infinity;
+  for (let i = sentence.globalIndex - 1; i >= 0; i--) {
+    const s = allSentences[i];
+    if (s === void 0) continue;
+    if (/^h[1-6]$/.test(s.blockType)) {
+      nearestHeading = s;
+      distance = sentence.globalIndex - i;
+      break;
+    }
+  }
+  if (nearestHeading === null) {
+    return 0.3;
+  }
+  const headingOverlap = termOverlapRatio(queryTokens, nearestHeading.tokens);
+  const distanceScore = 1 / (1 + distance / 5);
+  if (headingOverlap > 0) {
+    return Math.min(1, headingOverlap * 0.6 + distanceScore * 0.4);
+  }
+  return distanceScore * 0.5;
+}
+function calculateDensityScore(sentence, queryTokens) {
+  if (sentence.tokens.length === 0 || queryTokens.length === 0) {
+    return 0;
+  }
+  const querySet = new Set(queryTokens);
+  let queryTermCount = 0;
+  for (const token of sentence.tokens) {
+    if (querySet.has(token)) {
+      queryTermCount++;
+    }
+  }
+  const rawDensity = queryTermCount / sentence.tokens.length;
+  const uniqueQueryTerms = /* @__PURE__ */ new Set();
+  for (const token of sentence.tokens) {
+    if (querySet.has(token)) {
+      uniqueQueryTerms.add(token);
+    }
+  }
+  const coverage = uniqueQueryTerms.size / queryTokens.length;
+  return rawDensity * 0.4 + coverage * 0.6;
+}
+function calculateStructureScore(sentence, queryTokens, allSentences) {
+  let baseScore;
+  switch (sentence.blockType) {
+    case "p":
+      baseScore = 0.8;
+      break;
+    case "li":
+      baseScore = 0.7;
+      break;
+    case "pre":
+      baseScore = 0.65;
+      break;
+    default:
+      baseScore = 0.4;
+  }
+  const codeAdjacentBonus = sentence.blockType !== "pre" && allSentences.some(
+    (s) => s.blockType === "pre" && s.globalIndex !== sentence.globalIndex && Math.abs(s.globalIndex - sentence.globalIndex) <= 2
+  ) ? 0.1 : 0;
+  let sameBlockBonus = 0;
+  for (const s of allSentences) {
+    if (s.blockIndex === sentence.blockIndex && s.globalIndex !== sentence.globalIndex) {
+      const overlap = termOverlapRatio(queryTokens, s.tokens);
+      if (overlap > 0.3) {
+        sameBlockBonus = Math.max(sameBlockBonus, 0.15);
+      }
+    }
+  }
+  let headingPathBonus = 0;
+  for (const heading of sentence.headingPath) {
+    const headingLower = heading.toLowerCase();
+    for (const token of queryTokens) {
+      if (headingLower.includes(token)) {
+        headingPathBonus = 0.1;
+        break;
+      }
+    }
+    if (headingPathBonus > 0) break;
+  }
+  return Math.min(1, baseScore + sameBlockBonus + headingPathBonus + codeAdjacentBonus);
+}
+function calculateProximityScore(sentence, queryTokens) {
+  if (sentence.tokens.length === 0 || queryTokens.length === 0) {
+    return 0;
+  }
+  const querySet = new Set(queryTokens);
+  const termPositions = /* @__PURE__ */ new Map();
+  for (let i = 0; i < sentence.tokens.length; i++) {
+    const token = sentence.tokens[i];
+    if (token !== void 0 && querySet.has(token)) {
+      const positions = termPositions.get(token);
+      if (positions === void 0) {
+        termPositions.set(token, [i]);
+      } else {
+        positions.push(i);
+      }
+    }
+  }
+  const matchedTermCount = termPositions.size;
+  if (matchedTermCount === 0) {
+    return 0;
+  }
+  const coverageRatio = matchedTermCount / queryTokens.length;
+  if (matchedTermCount === 1) {
+    return coverageRatio * 0.5;
+  }
+  const allPositions = [];
+  for (const [term, positions] of termPositions) {
+    for (const pos of positions) {
+      allPositions.push({ pos, term });
+    }
+  }
+  allPositions.sort((a, b) => a.pos - b.pos);
+  let minSpan = Infinity;
+  const windowTermCount = /* @__PURE__ */ new Map();
+  let uniqueTermsInWindow = 0;
+  let left = 0;
+  for (let right = 0; right < allPositions.length; right++) {
+    const rightItem = allPositions[right];
+    if (rightItem === void 0) continue;
+    const rightTerm = rightItem.term;
+    const prevCount = windowTermCount.get(rightTerm) ?? 0;
+    windowTermCount.set(rightTerm, prevCount + 1);
+    if (prevCount === 0) {
+      uniqueTermsInWindow++;
+    }
+    while (uniqueTermsInWindow === matchedTermCount) {
+      const leftItem = allPositions[left];
+      if (leftItem === void 0) break;
+      const currentSpan = rightItem.pos - leftItem.pos + 1;
+      if (currentSpan < minSpan) {
+        minSpan = currentSpan;
+      }
+      const leftTerm = leftItem.term;
+      const leftCount = windowTermCount.get(leftTerm) ?? 0;
+      windowTermCount.set(leftTerm, leftCount - 1);
+      if (leftCount - 1 === 0) {
+        uniqueTermsInWindow--;
+      }
+      left++;
+    }
+  }
+  const spanTightness = 1 - Math.min(1, minSpan / sentence.tokens.length);
+  const densityInSpan = matchedTermCount / minSpan;
+  return coverageRatio * 0.4 + spanTightness * 0.35 + Math.min(1, densityInSpan) * 0.25;
+}
+function calculateHeadingPathScore(sentence, queryTokens, getIdf) {
+  if (sentence.headingPath.length === 0) {
+    return 0.3;
+  }
+  const headingText = sentence.headingPath.join(" ");
+  const headingTokens = tokenize(headingText);
+  if (headingTokens.length === 0 || queryTokens.length === 0) {
+    return 0.3;
+  }
+  const headingSet = new Set(headingTokens);
+  let matchedIdfSum = 0;
+  let totalIdfSum = 0;
+  for (const queryTerm of queryTokens) {
+    const idf = getIdf(queryTerm);
+    totalIdfSum += idf;
+    if (headingSet.has(queryTerm)) {
+      matchedIdfSum += idf;
+    }
+  }
+  if (totalIdfSum === 0) {
+    return 0.3;
+  }
+  const idfWeightedOverlap = matchedIdfSum / totalIdfSum;
+  return 0.3 + idfWeightedOverlap * 0.7;
+}
+function calculateCoverageScore(sentence, queryTokens, getIdf) {
+  if (sentence.tokens.length === 0 || queryTokens.length === 0) {
+    return 0;
+  }
+  const sentenceSet = new Set(sentence.tokens);
+  let matchedIdfSum = 0;
+  let totalIdfSum = 0;
+  let matchedCount = 0;
+  for (const queryTerm of queryTokens) {
+    const idf = getIdf(queryTerm);
+    totalIdfSum += idf;
+    if (sentenceSet.has(queryTerm)) {
+      matchedIdfSum += idf;
+      matchedCount++;
+    }
+  }
+  if (totalIdfSum === 0) {
+    return 0;
+  }
+  const idfWeightedCoverage = matchedIdfSum / totalIdfSum;
+  const simpleCoverage = matchedCount / queryTokens.length;
+  return idfWeightedCoverage * 0.7 + simpleCoverage * 0.3;
+}
+function computeDensityStats(sentences, queryTokens) {
+  if (sentences.length === 0 || queryTokens.length === 0) {
+    return { median: 0, mad: 1e-3 };
+  }
+  const querySet = new Set(queryTokens);
+  const densities = [];
+  for (const sentence of sentences) {
+    if (sentence.tokens.length === 0) {
+      densities.push(0);
+      continue;
+    }
+    let queryTermCount = 0;
+    for (const token of sentence.tokens) {
+      if (querySet.has(token)) {
+        queryTermCount++;
+      }
+    }
+    densities.push(queryTermCount / sentence.tokens.length);
+  }
+  densities.sort((a, b) => a - b);
+  const mid = Math.floor(densities.length / 2);
+  const median = densities.length % 2 === 0 ? ((densities[mid - 1] ?? 0) + (densities[mid] ?? 0)) / 2 : densities[mid] ?? 0;
+  const absoluteDeviations = densities.map((d) => Math.abs(d - median));
+  absoluteDeviations.sort((a, b) => a - b);
+  const madMid = Math.floor(absoluteDeviations.length / 2);
+  let mad = absoluteDeviations.length % 2 === 0 ? ((absoluteDeviations[madMid - 1] ?? 0) + (absoluteDeviations[madMid] ?? 0)) / 2 : absoluteDeviations[madMid] ?? 0;
+  if (mad < 1e-3) {
+    mad = 1e-3;
+  }
+  return { median, mad };
+}
+function isMetaHeading(heading) {
+  const trimmed = heading.trim();
+  for (const pattern of META_SECTION_PATTERNS) {
+    if (pattern.test(trimmed)) {
+      return true;
+    }
+  }
+  return false;
+}
+function calculateMetaSectionScore(sentence) {
+  for (const heading of sentence.headingPath) {
+    if (isMetaHeading(heading)) {
+      return 0.2;
+    }
+  }
+  const text = sentence.text.toLowerCase();
+  const metaPhrases = [
+    "in this article",
+    "in this guide",
+    "in this tutorial",
+    "in this post",
+    "we will cover",
+    "we'll cover",
+    "you will learn",
+    "you'll learn",
+    "we will explore",
+    "we'll explore",
+    "let's explore",
+    "let's dive",
+    "let's get started",
+    "i will show you",
+    "i'll show you",
+    "we will discuss",
+    "we'll discuss",
+    "this article covers",
+    "this guide covers",
+    "this tutorial covers",
+    "by the end of this",
+    "after reading this",
+    "if you enjoyed this",
+    "if you found this helpful",
+    "don't forget to",
+    "make sure to subscribe",
+    "leave a comment",
+    "share this article",
+    "follow me on",
+    "check out my",
+    "support my work",
+    "buy me a coffee"
+  ];
+  for (const phrase of metaPhrases) {
+    if (text.includes(phrase)) {
+      return 0.3;
+    }
+  }
+  return 1;
+}
+function calculateOutlierScore(sentence, queryTokens, densityStats) {
+  if (sentence.tokens.length === 0 || queryTokens.length === 0) {
+    return 0.3;
+  }
+  const querySet = new Set(queryTokens);
+  let queryTermCount = 0;
+  for (const token of sentence.tokens) {
+    if (querySet.has(token)) {
+      queryTermCount++;
+    }
+  }
+  const density = queryTermCount / sentence.tokens.length;
+  const zScore = (density - densityStats.median) / densityStats.mad;
+  if (zScore <= 0) {
+    return 0.3;
+  }
+  const sigmoid = 1 / (1 + Math.exp(-(zScore - 2)));
+  return 0.3 + 0.7 * sigmoid;
+}
+function calculateHeuristicScores(sentence, queryTokens, allSentences, getIdf, densityStats, weights = {}) {
+  const w = { ...DEFAULT_WEIGHTS, ...weights };
+  const positionScore = calculatePositionScore(sentence);
+  const headingProximityScore = calculateHeadingProximityScore(sentence, queryTokens, allSentences);
+  const densityScore = calculateDensityScore(sentence, queryTokens);
+  const structureScore = calculateStructureScore(sentence, queryTokens, allSentences);
+  const proximityScore = calculateProximityScore(sentence, queryTokens);
+  const headingPathScore = calculateHeadingPathScore(sentence, queryTokens, getIdf);
+  const coverageScore = calculateCoverageScore(sentence, queryTokens, getIdf);
+  const outlierScore = calculateOutlierScore(sentence, queryTokens, densityStats);
+  const metaSectionScore = calculateMetaSectionScore(sentence);
+  const combined = w.position * positionScore + w.headingProximity * headingProximityScore + w.density * densityScore + w.structure * structureScore + w.proximity * proximityScore + w.headingPath * headingPathScore + w.coverage * coverageScore + w.outlier * outlierScore + w.metaSection * metaSectionScore;
+  return {
+    positionScore,
+    headingProximityScore,
+    densityScore,
+    structureScore,
+    proximityScore,
+    headingPathScore,
+    coverageScore,
+    outlierScore,
+    metaSectionScore,
+    combined
+  };
+}
+function scoreAllSentencesHeuristics(sentences, queryTokens, getIdf, weights = {}) {
+  const scores = /* @__PURE__ */ new Map();
+  const densityStats = computeDensityStats(sentences, queryTokens);
+  for (const sentence of sentences) {
+    const heuristics = calculateHeuristicScores(
+      sentence,
+      queryTokens,
+      sentences,
+      getIdf,
+      densityStats,
+      weights
+    );
+    scores.set(sentence.globalIndex, heuristics);
+  }
+  return scores;
+}
+
+// src/scoring/ranker.ts
+var DEFAULT_CONFIG2 = {
+  bm25Weight: 0.6,
+  heuristicWeight: 0.4,
+  relevanceMode: "strict"
+};
+var SEARCH_RELEVANCE_THRESHOLDS = {
+  strongSentenceBm25: 0.8,
+  // Was 1.2
+  strongSentenceCoverage: 0.25,
+  cooccurrenceMinTerms: 2,
+  cooccurrenceBm25: 0.5,
+  // Was 0.8
+  centralTermBm25: 0.4,
+  // Was 0.6
+  goodCoverage: 0.5,
+  goodCoverageBm25: 0.3
+  // Was 0.4
+};
+var STRICT_RELEVANCE_THRESHOLDS = {
+  cooccurrenceMinTerms: 2,
+  cooccurrenceBm25: 1,
+  centralTermBm25: 0.8,
+  highCoverage: 0.8,
+  highCoverageBm25: 0.5
+};
+function normalizeScores(scores) {
+  if (scores.size === 0) return /* @__PURE__ */ new Map();
+  let min = Infinity;
+  let max = -Infinity;
+  for (const score of scores.values()) {
+    if (score < min) min = score;
+    if (score > max) max = score;
+  }
+  const range = max - min;
+  const normalized = /* @__PURE__ */ new Map();
+  for (const [id, score] of scores) {
+    normalized.set(id, range === 0 ? 0.5 : (score - min) / range);
+  }
+  return normalized;
+}
+function rankSentencesWithRelevance(sentences, queryTokens, config = {}) {
+  const {
+    bm25Weight = DEFAULT_CONFIG2.bm25Weight,
+    heuristicWeight = DEFAULT_CONFIG2.heuristicWeight,
+    bm25Config,
+    heuristicWeights,
+    relevanceMode = DEFAULT_CONFIG2.relevanceMode
+  } = config;
+  if (sentences.length === 0 || queryTokens.length === 0) {
+    return {
+      sentences: sentences.map((s) => ({
+        ...s,
+        bm25Score: 0,
+        heuristicScore: 0,
+        combinedScore: 0
+      })),
+      hasRelevantResults: false,
+      maxRawBm25: 0,
+      queryTermCoverage: 0,
+      maxCooccurrence: 0
+    };
+  }
+  const bm25Scorer = createBM25Scorer(sentences, bm25Config);
+  const rawBm25Scores = /* @__PURE__ */ new Map();
+  let maxRawBm25 = 0;
+  for (const sentence of sentences) {
+    const docTf = buildTermFrequencyMap(sentence.tokens);
+    const score = bm25Scorer.scoreWithTf(queryTokens, docTf, sentence.tokens.length);
+    rawBm25Scores.set(sentence.globalIndex, score);
+    if (score > maxRawBm25) {
+      maxRawBm25 = score;
+    }
+  }
+  const queryTermSet = new Set(queryTokens);
+  const foundTerms = /* @__PURE__ */ new Set();
+  for (const sentence of sentences) {
+    for (const token of sentence.tokens) {
+      if (queryTermSet.has(token)) {
+        foundTerms.add(token);
+      }
+    }
+  }
+  const queryTermCoverage = queryTokens.length > 0 ? foundTerms.size / queryTokens.length : 0;
+  let maxCooccurrence = 0;
+  const termSentenceCount = /* @__PURE__ */ new Map();
+  for (const sentence of sentences) {
+    const uniqueInSentence = /* @__PURE__ */ new Set();
+    for (const token of sentence.tokens) {
+      if (queryTermSet.has(token)) {
+        uniqueInSentence.add(token);
+      }
+    }
+    if (uniqueInSentence.size > maxCooccurrence) {
+      maxCooccurrence = uniqueInSentence.size;
+    }
+    for (const term of uniqueInSentence) {
+      termSentenceCount.set(term, (termSentenceCount.get(term) ?? 0) + 1);
+    }
+  }
+  const centralTermThreshold = Math.max(3, sentences.length * 0.1);
+  let hasCentralTerm = false;
+  for (const count of termSentenceCount.values()) {
+    if (count >= centralTermThreshold) {
+      hasCentralTerm = true;
+      break;
+    }
+  }
+  const hasRelevantResults = relevanceMode === "search" ? (
+    // Single strong sentence with decent coverage (search mode)
+    maxRawBm25 > SEARCH_RELEVANCE_THRESHOLDS.strongSentenceBm25 && queryTermCoverage >= SEARCH_RELEVANCE_THRESHOLDS.strongSentenceCoverage || // Multiple query terms in same sentence
+    maxCooccurrence >= SEARCH_RELEVANCE_THRESHOLDS.cooccurrenceMinTerms && maxRawBm25 > SEARCH_RELEVANCE_THRESHOLDS.cooccurrenceBm25 || // A matching term is central to the document
+    hasCentralTerm && maxRawBm25 > SEARCH_RELEVANCE_THRESHOLDS.centralTermBm25 || // Good coverage
+    queryTermCoverage >= SEARCH_RELEVANCE_THRESHOLDS.goodCoverage && maxRawBm25 > SEARCH_RELEVANCE_THRESHOLDS.goodCoverageBm25
+  ) : (
+    // Multiple query terms in same sentence with good BM25 (strict mode)
+    maxCooccurrence >= STRICT_RELEVANCE_THRESHOLDS.cooccurrenceMinTerms && maxRawBm25 > STRICT_RELEVANCE_THRESHOLDS.cooccurrenceBm25 || // A matching term is central to the document (topic match)
+    hasCentralTerm && maxRawBm25 > STRICT_RELEVANCE_THRESHOLDS.centralTermBm25 || // Very high coverage
+    queryTermCoverage >= STRICT_RELEVANCE_THRESHOLDS.highCoverage && maxRawBm25 > STRICT_RELEVANCE_THRESHOLDS.highCoverageBm25
+  );
+  const heuristicScores = scoreAllSentencesHeuristics(
+    sentences,
+    queryTokens,
+    bm25Scorer.getIdf,
+    heuristicWeights
+  );
+  const normalizedBm25 = normalizeScores(rawBm25Scores);
+  const scoredSentences = sentences.map((sentence) => {
+    const bm25Score = normalizedBm25.get(sentence.globalIndex) ?? 0;
+    const heuristics = heuristicScores.get(sentence.globalIndex);
+    const heuristicScore = heuristics?.combined ?? 0;
+    const combinedScore = bm25Weight * bm25Score + heuristicWeight * heuristicScore;
+    return {
+      ...sentence,
+      bm25Score,
+      heuristicScore,
+      combinedScore
+    };
+  });
+  scoredSentences.sort((a, b) => {
+    const d = b.combinedScore - a.combinedScore;
+    if (d !== 0) return d;
+    return a.globalIndex - b.globalIndex;
+  });
+  return {
+    sentences: scoredSentences,
+    hasRelevantResults,
+    maxRawBm25,
+    queryTermCoverage,
+    maxCooccurrence
+  };
+}
+
+// src/extraction/anchors.ts
+var DEFAULT_CONFIG3 = {
+  maxAnchors: 5,
+  minScore: 0.1,
+  diversityThreshold: 0.5
+  // Max Jaccard similarity to already-selected anchors
+};
+function selectAnchorsWithPositionDiversity(rankedSentences, config = {}) {
+  const {
+    maxAnchors = DEFAULT_CONFIG3.maxAnchors,
+    minScore = DEFAULT_CONFIG3.minScore,
+    diversityThreshold = DEFAULT_CONFIG3.diversityThreshold,
+    minPositionGap = 3
+    // Minimum global index gap between anchors
+  } = config;
+  const anchors = [];
+  for (const sentence of rankedSentences) {
+    if (sentence.combinedScore < minScore) {
+      continue;
+    }
+    let tooSimilar = false;
+    for (const anchor of anchors) {
+      const similarity = jaccardSimilarity(sentence.tokens, anchor.tokens);
+      if (similarity > diversityThreshold) {
+        tooSimilar = true;
+        break;
+      }
+      const positionGap = Math.abs(sentence.globalIndex - anchor.globalIndex);
+      if (positionGap < minPositionGap) {
+        tooSimilar = true;
+        break;
+      }
+    }
+    if (!tooSimilar) {
+      anchors.push(sentence);
+      if (anchors.length >= maxAnchors) {
+        break;
+      }
+    }
+  }
+  return anchors;
+}
+
+// src/extraction/expand.ts
+function buildChunkText(sentences) {
+  const textParts = [];
+  let lastBlockType = "";
+  for (const s of sentences) {
+    if (/^h[1-6]$/.test(s.blockType) && textParts.length > 0) {
+      textParts.push("\n\n" + s.text);
+    } else if (s.blockType === "pre") {
+      textParts.push("\n\n```\n" + s.text + "\n```");
+    } else if (s.blockType === "li") {
+      if (lastBlockType !== "li") {
+        textParts.push("\n");
+      }
+      textParts.push("\n- " + s.text);
+    } else {
+      if (lastBlockType === "pre" || /^h[1-6]$/.test(lastBlockType)) {
+        textParts.push("\n\n" + s.text);
+      } else if (textParts.length > 0) {
+        textParts.push(" " + s.text);
+      } else {
+        textParts.push(s.text);
+      }
+    }
+    lastBlockType = s.blockType;
+  }
+  return textParts.join("").trim();
+}
+var DEFAULT_CONFIG4 = {
+  contextBefore: 5,
+  contextAfter: 8,
+  respectBlockBoundaries: false,
+  // Changed: allow crossing blocks by default
+  maxChunkChars: 2e3,
+  includeCodeBlocks: true,
+  expandToSection: true
+};
+function findSectionBoundaries(anchorIndex, allSentences, maxChars) {
+  const anchor = allSentences[anchorIndex];
+  if (anchor === void 0) {
+    return { start: anchorIndex, end: anchorIndex };
+  }
+  let sectionHeadingLevel = 6;
+  let sectionStart = anchorIndex;
+  for (let i = anchorIndex; i >= 0; i--) {
+    const s = allSentences[i];
+    if (s === void 0) continue;
+    if (/^h[1-6]$/.test(s.blockType)) {
+      const level = parseInt(s.blockType[1] ?? "6", 10);
+      sectionHeadingLevel = level;
+      sectionStart = i;
+      break;
+    }
+  }
+  let sectionEnd = allSentences.length - 1;
+  for (let i = anchorIndex + 1; i < allSentences.length; i++) {
+    const s = allSentences[i];
+    if (s === void 0) continue;
+    if (/^h[1-6]$/.test(s.blockType)) {
+      const level = parseInt(s.blockType[1] ?? "6", 10);
+      if (level <= sectionHeadingLevel) {
+        sectionEnd = i - 1;
+        break;
+      }
+    }
+  }
+  let charCount = 0;
+  let actualStart = sectionStart;
+  let actualEnd = sectionEnd;
+  for (let i = sectionStart; i <= sectionEnd; i++) {
+    const s = allSentences[i];
+    if (s !== void 0) {
+      charCount += s.text.length + 1;
+    }
+  }
+  if (charCount <= maxChars) {
+    return { start: sectionStart, end: sectionEnd };
+  }
+  charCount = allSentences[anchorIndex]?.text.length ?? 0;
+  actualStart = anchorIndex;
+  actualEnd = anchorIndex;
+  let canExpandBefore = actualStart > sectionStart;
+  let canExpandAfter = actualEnd < sectionEnd;
+  while ((canExpandBefore || canExpandAfter) && charCount < maxChars) {
+    if (canExpandBefore) {
+      const prev = allSentences[actualStart - 1];
+      if (prev && charCount + prev.text.length < maxChars) {
+        actualStart--;
+        charCount += prev.text.length + 1;
+        canExpandBefore = actualStart > sectionStart;
+      } else {
+        canExpandBefore = false;
+      }
+    }
+    if (canExpandAfter && charCount < maxChars) {
+      const next = allSentences[actualEnd + 1];
+      if (next && charCount + next.text.length < maxChars) {
+        actualEnd++;
+        charCount += next.text.length + 1;
+        canExpandAfter = actualEnd < sectionEnd;
+      } else {
+        canExpandAfter = false;
+      }
+    }
+  }
+  return { start: actualStart, end: actualEnd };
+}
+function expandToIncludeCode(endIndex, allSentences, maxChars, currentChars) {
+  let newEnd = endIndex;
+  let chars = currentChars;
+  for (let i = endIndex + 1; i < allSentences.length; i++) {
+    const s = allSentences[i];
+    if (s === void 0) break;
+    if (/^h[1-6]$/.test(s.blockType)) {
+      break;
+    }
+    if (s.blockType === "pre") {
+      if (chars + s.text.length <= maxChars * 1.5) {
+        newEnd = i;
+        chars += s.text.length;
+      }
+      break;
+    }
+    if (chars + s.text.length <= maxChars) {
+      newEnd = i;
+      chars += s.text.length;
+    } else {
+      break;
+    }
+  }
+  return newEnd;
+}
+function expandAnchor(anchor, allSentences, config = {}) {
+  const {
+    contextBefore = DEFAULT_CONFIG4.contextBefore,
+    contextAfter = DEFAULT_CONFIG4.contextAfter,
+    maxChunkChars = DEFAULT_CONFIG4.maxChunkChars,
+    includeCodeBlocks = DEFAULT_CONFIG4.includeCodeBlocks,
+    expandToSection = DEFAULT_CONFIG4.expandToSection
+  } = config;
+  const anchorIndex = anchor.globalIndex;
+  let startIndex;
+  let endIndex;
+  if (expandToSection) {
+    const bounds = findSectionBoundaries(anchorIndex, allSentences, maxChunkChars);
+    startIndex = bounds.start;
+    endIndex = bounds.end;
+  } else {
+    startIndex = Math.max(0, anchorIndex - contextBefore);
+    endIndex = Math.min(allSentences.length - 1, anchorIndex + contextAfter);
+    let charCount = 0;
+    for (let i = startIndex; i <= endIndex; i++) {
+      const s = allSentences[i];
+      if (s !== void 0) {
+        charCount += s.text.length;
+      }
+    }
+    while (charCount > maxChunkChars && startIndex < anchorIndex) {
+      const s = allSentences[startIndex];
+      if (s !== void 0) {
+        charCount -= s.text.length;
+      }
+      startIndex++;
+    }
+    while (charCount > maxChunkChars && endIndex > anchorIndex) {
+      const s = allSentences[endIndex];
+      if (s !== void 0) {
+        charCount -= s.text.length;
+      }
+      endIndex--;
+    }
+  }
+  if (includeCodeBlocks) {
+    let currentChars = 0;
+    for (let i = startIndex; i <= endIndex; i++) {
+      const s = allSentences[i];
+      if (s !== void 0) {
+        currentChars += s.text.length;
+      }
+    }
+    endIndex = expandToIncludeCode(endIndex, allSentences, maxChunkChars, currentChars);
+  }
+  const chunkSentences = [];
+  for (let i = startIndex; i <= endIndex; i++) {
+    const s = allSentences[i];
+    if (s !== void 0) {
+      chunkSentences.push(s);
+    }
+  }
+  const text = buildChunkText(chunkSentences);
+  const headingPath = anchor.headingPath;
+  const rawCharCount = chunkSentences.reduce((sum, s) => sum + s.text.length, 0);
+  return {
+    sentences: chunkSentences,
+    anchorIndex: anchor.globalIndex,
+    score: anchor.combinedScore,
+    text,
+    charCount: rawCharCount,
+    // Use raw content for budget, not formatted text
+    headingPath
+  };
+}
+function expandAnchors(anchors, allSentences, config = {}) {
+  return anchors.map((anchor) => expandAnchor(anchor, allSentences, config));
+}
+
+// src/extraction/dedupe.ts
+var DEFAULT_CONFIG5 = {
+  overlapThreshold: 0.5,
+  // Sentence index overlap ratio to trigger merge
+  tokenSimilarityThreshold: 0.72
+  // Token Jaccard similarity to consider duplicate
+};
+function calculateSentenceOverlap(chunkA, chunkB) {
+  const indicesA = new Set(chunkA.sentences.map((s) => s.globalIndex));
+  const indicesB = new Set(chunkB.sentences.map((s) => s.globalIndex));
+  let intersection = 0;
+  for (const idx of indicesA) {
+    if (indicesB.has(idx)) {
+      intersection++;
+    }
+  }
+  const smaller = Math.min(indicesA.size, indicesB.size);
+  if (smaller === 0) return 0;
+  return intersection / smaller;
+}
+function mergeChunks(chunkA, chunkB) {
+  const sentenceMap = /* @__PURE__ */ new Map();
+  for (const s of chunkA.sentences) {
+    sentenceMap.set(s.globalIndex, s);
+  }
+  for (const s of chunkB.sentences) {
+    sentenceMap.set(s.globalIndex, s);
+  }
+  const mergedSentences = Array.from(sentenceMap.values()).sort((a, b) => a.globalIndex - b.globalIndex);
+  const keepChunk = chunkA.score >= chunkB.score ? chunkA : chunkB;
+  const text = buildChunkText(mergedSentences);
+  return {
+    sentences: mergedSentences,
+    anchorIndex: keepChunk.anchorIndex,
+    score: Math.max(chunkA.score, chunkB.score),
+    text,
+    charCount: text.length,
+    headingPath: keepChunk.headingPath
+  };
+}
+function getChunkTokens(chunk) {
+  const tokens = [];
+  for (const s of chunk.sentences) {
+    tokens.push(...s.tokens);
+  }
+  return tokens;
+}
+function dedupeChunks(chunks, config = {}) {
+  const {
+    overlapThreshold = DEFAULT_CONFIG5.overlapThreshold,
+    tokenSimilarityThreshold = DEFAULT_CONFIG5.tokenSimilarityThreshold
+  } = config;
+  if (chunks.length <= 1) {
+    return chunks;
+  }
+  const sorted = [...chunks].sort((a, b) => {
+    const d = b.score - a.score;
+    if (d !== 0) return d;
+    return a.anchorIndex - b.anchorIndex;
+  });
+  const result = [];
+  const merged = /* @__PURE__ */ new Set();
+  for (let i = 0; i < sorted.length; i++) {
+    if (merged.has(i)) continue;
+    let accumulatedChunk = sorted[i];
+    if (accumulatedChunk === void 0) continue;
+    for (let j = i + 1; j < sorted.length; j++) {
+      if (merged.has(j)) continue;
+      const other = sorted[j];
+      if (other === void 0) continue;
+      const overlap = calculateSentenceOverlap(accumulatedChunk, other);
+      if (overlap >= overlapThreshold) {
+        accumulatedChunk = mergeChunks(accumulatedChunk, other);
+        merged.add(j);
+        continue;
+      }
+      const accumulatedTokens = getChunkTokens(accumulatedChunk);
+      const otherTokens = getChunkTokens(other);
+      const similarity = jaccardSimilarity(accumulatedTokens, otherTokens);
+      if (similarity >= tokenSimilarityThreshold) {
+        merged.add(j);
+      }
+    }
+    result.push(accumulatedChunk);
+  }
+  result.sort((a, b) => {
+    const d = b.score - a.score;
+    if (d !== 0) return d;
+    return a.anchorIndex - b.anchorIndex;
+  });
+  return result;
+}
+function removeSubsetChunks(chunks) {
+  if (chunks.length <= 1) {
+    return chunks;
+  }
+  const sorted = [...chunks].sort((a, b) => {
+    const d = b.sentences.length - a.sentences.length;
+    if (d !== 0) return d;
+    return a.anchorIndex - b.anchorIndex;
+  });
+  const result = [];
+  for (const chunk of sorted) {
+    let isSubset = false;
+    for (const existing of result) {
+      const existingIndices = new Set(existing.sentences.map((s) => s.globalIndex));
+      const chunkIndices = chunk.sentences.map((s) => s.globalIndex);
+      const allInExisting = chunkIndices.every((idx) => existingIndices.has(idx));
+      if (allInExisting) {
+        isSubset = true;
+        break;
+      }
+    }
+    if (!isSubset) {
+      result.push(chunk);
+    }
+  }
+  result.sort((a, b) => {
+    const d = b.score - a.score;
+    if (d !== 0) return d;
+    return a.anchorIndex - b.anchorIndex;
+  });
+  return result;
+}
+function fullDedupe(chunks, config = {}) {
+  const merged = dedupeChunks(chunks, config);
+  return removeSubsetChunks(merged);
+}
+
+// src/pipeline.ts
+function createEmptyResult(query, debug, debugInfo, relevanceMetrics) {
+  const result = {
+    excerpts: [],
+    totalChars: 0,
+    query
+  };
+  if (relevanceMetrics) {
+    result.relevanceMetrics = relevanceMetrics;
+  }
+  if (debug) {
+    result.debug = {
+      sentenceCount: 0,
+      anchorCount: 0,
+      chunkCount: 0,
+      dedupedChunkCount: 0,
+      topSentences: [],
+      ...debugInfo
+    };
+  }
+  return result;
+}
+var DEFAULT_CONFIG6 = {
+  ranker: {
+    bm25Weight: 0.6,
+    heuristicWeight: 0.4
+  },
+  anchors: {
+    maxAnchors: 5,
+    minScore: 0.25,
+    // Higher threshold to filter irrelevant results
+    diversityThreshold: 0.4,
+    minPositionGap: 5
+  },
+  expand: {
+    contextBefore: 5,
+    contextAfter: 8,
+    respectBlockBoundaries: false,
+    maxChunkChars: 2e3,
+    includeCodeBlocks: true,
+    expandToSection: true
+  },
+  dedupe: {
+    overlapThreshold: 0.3,
+    tokenSimilarityThreshold: 0.6
+  },
+  excerpts: {
+    maxExcerpts: 3,
+    charBudget: 6e3,
+    minExcerptChars: 100
+  },
+  debug: false
+};
+function extractExcerpts(html, query, config = {}) {
+  const cfg = mergeConfig(DEFAULT_CONFIG6, config);
+  const logger2 = logger_default.getInstance();
+  const { $, mainContent } = logger2.time("1. Preprocess HTML", () => preprocessHtml(html));
+  if (mainContent === null) {
+    return createEmptyResult(query, cfg.debug);
+  }
+  const { sentences } = logger2.time("2. Segment into sentences", () => segmentHtml($, mainContent));
+  if (sentences.length === 0) {
+    return createEmptyResult(query, cfg.debug);
+  }
+  const queryTokens = logger2.time("3. Tokenize query", () => tokenize(query));
+  if (queryTokens.length === 0) {
+    return extractWithoutQuery(sentences, query, cfg);
+  }
+  const rankingResult = logger2.time("4. Rank sentences", () => rankSentencesWithRelevance(sentences, queryTokens, cfg.ranker));
+  const rankedSentences = rankingResult.sentences;
+  if (!rankingResult.hasRelevantResults) {
+    return createEmptyResult(
+      query,
+      cfg.debug,
+      {
+        sentenceCount: sentences.length,
+        queryTermCoverage: rankingResult.queryTermCoverage,
+        maxRawBm25: rankingResult.maxRawBm25,
+        hasRelevantResults: false,
+        topSentences: rankedSentences.slice(0, 5).map((s) => ({
+          text: s.text.slice(0, 100) + (s.text.length > 100 ? "..." : ""),
+          score: s.combinedScore,
+          headingPath: s.headingPath
+        }))
+      },
+      {
+        hasRelevantResults: false,
+        sentenceCount: sentences.length,
+        queryTermCoverage: rankingResult.queryTermCoverage,
+        maxBm25: rankingResult.maxRawBm25,
+        maxCooccurrence: rankingResult.maxCooccurrence
+      }
+    );
+  }
+  const anchors = logger2.time("5. Select anchors", () => selectAnchorsWithPositionDiversity(rankedSentences, cfg.anchors));
+  const chunks = logger2.time("6. Expand anchors", () => expandAnchors(anchors, sentences, cfg.expand));
+  const dedupedChunks = logger2.time("7. Deduplicate chunks", () => fullDedupe(chunks, cfg.dedupe));
+  const result = logger2.time("8. Assemble excerpts", () => assembleExcerpts(dedupedChunks, query, cfg.excerpts));
+  const relevanceMetrics = {
+    hasRelevantResults: rankingResult.hasRelevantResults,
+    sentenceCount: sentences.length,
+    queryTermCoverage: rankingResult.queryTermCoverage,
+    maxBm25: rankingResult.maxRawBm25,
+    maxCooccurrence: rankingResult.maxCooccurrence
+  };
+  if (cfg.debug) {
+    const debugInfo = {
+      sentenceCount: sentences.length,
+      anchorCount: anchors.length,
+      chunkCount: chunks.length,
+      dedupedChunkCount: dedupedChunks.length,
+      queryTermCoverage: rankingResult.queryTermCoverage,
+      maxRawBm25: rankingResult.maxRawBm25,
+      hasRelevantResults: rankingResult.hasRelevantResults,
+      topSentences: rankedSentences.slice(0, 10).map((s) => ({
+        text: s.text.slice(0, 100) + (s.text.length > 100 ? "..." : ""),
+        score: s.combinedScore,
+        headingPath: s.headingPath
+      }))
+    };
+    return { ...result, debug: debugInfo, relevanceMetrics };
+  }
+  return { ...result, relevanceMetrics };
+}
+function extractWithoutQuery(sentences, query, cfg) {
+  const earlySentences = sentences.filter((s) => s.position < 0.4).slice(0, cfg.anchors?.maxAnchors ?? 5);
+  if (earlySentences.length === 0) {
+    return {
+      excerpts: [],
+      totalChars: 0,
+      query
+    };
+  }
+  const excerpts = earlySentences.map((s) => ({
+    text: s.text,
+    headingPath: s.headingPath,
+    score: 1 - s.position,
+    // Higher score for earlier content
+    charCount: s.text.length
+  }));
+  let totalChars = 0;
+  const budget = cfg.excerpts?.charBudget ?? 2e3;
+  const maxExcerpts = cfg.excerpts?.maxExcerpts ?? 3;
+  const selected = [];
+  for (const excerpt of excerpts) {
+    if (totalChars + excerpt.charCount > budget) break;
+    if (selected.length >= maxExcerpts) break;
+    selected.push(excerpt);
+    totalChars += excerpt.charCount;
+  }
+  return {
+    excerpts: selected,
+    totalChars,
+    query
+  };
+}
+function mergeConfig(defaults, overrides) {
+  return {
+    ranker: { ...defaults.ranker, ...overrides.ranker },
+    anchors: { ...defaults.anchors, ...overrides.anchors },
+    expand: { ...defaults.expand, ...overrides.expand },
+    dedupe: { ...defaults.dedupe, ...overrides.dedupe },
+    excerpts: { ...defaults.excerpts, ...overrides.excerpts },
+    debug: overrides.debug ?? defaults.debug ?? false
+  };
+}
+
+// src/mcp/types.ts
+var DEFAULT_CONFIG7 = {
+  searxngUrl: "http://localhost:8888",
+  maxResults: 5,
+  timeout: 5e3,
+  perPageCharBudget: 3e3,
+  // Increased for docs pages
+  totalCharBudget: 12e3
+  // Increased to accommodate larger excerpts
+};
+
+// src/mcp/searxng.ts
+async function searchSearxng(query, options) {
+  const { baseUrl, maxResults, timeout = 1e4 } = options;
+  const searchUrl = new URL("/search", baseUrl);
+  searchUrl.searchParams.set("q", query);
+  searchUrl.searchParams.set("format", "json");
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const response = await fetch(searchUrl.toString(), {
+      signal: controller.signal,
+      headers: {
+        "Accept": "application/json"
+      }
+    });
+    if (!response.ok) {
+      throw new Error(`SearXNG returned ${response.status}: ${response.statusText}`);
+    }
+    const data = await response.json();
+    return data.results.slice(0, maxResults).map((r) => ({
+      url: r.url,
+      title: r.title ?? "",
+      content: r.content ?? "",
+      score: r.score ?? 1,
+      engine: r.engine ?? "unknown"
+    }));
+  } catch (error) {
+    if (error instanceof Error && error.name === "AbortError") {
+      throw new Error(`SearXNG request timed out after ${timeout}ms`);
+    }
+    throw error;
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+
+// src/mcp/scraper.ts
+var DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; PeekyBot/1.0; +https://github.com/peeky-search)";
+var DEFAULT_MAX_CONCURRENT = 3;
+async function runWithConcurrency(items, fn, maxConcurrent) {
+  const queue = [...items];
+  const workers = [];
+  for (let i = 0; i < Math.min(maxConcurrent, queue.length); i++) {
+    workers.push((async () => {
+      while (queue.length > 0) {
+        const item = queue.shift();
+        if (item !== void 0) {
+          await fn(item);
+        }
+      }
+    })());
+  }
+  await Promise.all(workers);
+}
+async function scrapeUrl(url, options) {
+  const { timeout = 5e3, userAgent = DEFAULT_USER_AGENT } = options;
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: {
+        "User-Agent": userAgent,
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5"
+      },
+      redirect: "follow"
+    });
+    if (!response.ok) {
+      return {
+        url,
+        html: null,
+        error: `HTTP ${response.status}: ${response.statusText}`
+      };
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) {
+      return {
+        url,
+        html: null,
+        error: `Non-HTML content type: ${contentType}`
+      };
+    }
+    const html = await response.text();
+    return { url, html };
+  } catch (error) {
+    if (error instanceof Error) {
+      if (error.name === "AbortError") {
+        return {
+          url,
+          html: null,
+          error: `Timeout after ${timeout}ms`
+        };
+      }
+      return {
+        url,
+        html: null,
+        error: error.message
+      };
+    }
+    return {
+      url,
+      html: null,
+      error: "Unknown error"
+    };
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+async function scrapeUrls(urls, options = {}) {
+  const { maxConcurrent = DEFAULT_MAX_CONCURRENT } = options;
+  const results = [];
+  await runWithConcurrency(
+    urls,
+    async (url) => {
+      const result = await scrapeUrl(url, options);
+      results.push(result);
+    },
+    maxConcurrent
+  );
+  return results;
+}
+
+// src/mcp/query-parser.ts
+function parseSearchOperators(query) {
+  let extractionQuery = query;
+  extractionQuery = extractionQuery.replace(/\bsite:[\w.-]+/gi, "");
+  extractionQuery = extractionQuery.replace(/\s-\w+/g, "");
+  extractionQuery = extractionQuery.replace(/\bfiletype:\w+/gi, "");
+  extractionQuery = extractionQuery.replace(/"([^"]+)"/g, "$1");
+  extractionQuery = extractionQuery.replace(/\s+/g, " ").trim();
+  return {
+    searchQuery: query,
+    extractionQuery
+  };
+}
+
+// src/mcp/orchestrator.ts
+var logger = logger_default.getInstance();
+var JS_RENDERED_DOMAINS = /* @__PURE__ */ new Set([
+  "medium.com",
+  "npmjs.com"
+]);
+function isBlockedDomain(url) {
+  try {
+    const hostname = new URL(url).hostname.replace(/^www\./, "");
+    for (const blocked of JS_RENDERED_DOMAINS) {
+      if (hostname === blocked || hostname.endsWith(`.${blocked}`)) {
+        return true;
+      }
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}
+var VERSION_PATH_PATTERNS = [
+  /\/v\d+(\.\d+)*\//gi,
+  // /v1/, /v2/, /v1.2.3/
+  /\/\d+\.\d+(\.\d+)*\//gi,
+  // /4.3/, /3.0.0/
+  /\/(stable|latest|current|master|main|dev|nightly)\//gi,
+  /\/(en|en-us|en-gb)\//gi
+  // Language prefixes (often redundant)
+];
+var VERSION_SUBDOMAIN_PATTERNS = [
+  /^v\d+\./i,
+  // v1., v2., etc.
+  /^\d+\.\d+\./,
+  // 4.3., 3.0., etc.
+  /^(stable|latest|current|docs)\./i,
+  /^[a-z]{2}\./i,
+  // Two-letter language codes: en., he., fr., etc.
+  /^[a-z]{2}-[a-z]{2}\./i
+  // Regional codes: en-us., pt-br., etc.
+];
+function normalizeUrlForDedup(url) {
+  try {
+    const urlObj = new URL(url);
+    let hostname = urlObj.hostname;
+    let path = urlObj.pathname;
+    for (const pattern of VERSION_SUBDOMAIN_PATTERNS) {
+      hostname = hostname.replace(pattern, "");
+    }
+    for (const pattern of VERSION_PATH_PATTERNS) {
+      path = path.replace(pattern, "/");
+    }
+    path = path.replace(/\/+/g, "/");
+    return `${hostname}${path}`;
+  } catch {
+    return url;
+  }
+}
+function deduplicateUrls(results) {
+  const seen = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    const normalized = normalizeUrlForDedup(result.url);
+    if (!seen.has(normalized)) {
+      seen.set(normalized, result);
+    }
+  }
+  return Array.from(seen.values());
+}
+var PAGE_RELEVANCE_WEIGHTS = {
+  titleMatch: 0.35,
+  urlMatch: 0.15,
+  excerptScore: 0.35,
+  searxngScore: 0.15
+};
+var SEARCH_PIPELINE_CONFIG = {
+  ranker: {
+    relevanceMode: "search"
+    // Looser relevance for multi-page search
+  },
+  anchors: {
+    maxAnchors: 5
+    // More anchors for comprehensive docs coverage
+  },
+  expand: {
+    maxChunkChars: 2e3,
+    // Larger chunks for better context
+    contextAfter: 12,
+    // More trailing context for code explanations
+    contextBefore: 8
+    // More leading context
+  }
+};
+function extractTitle(html) {
+  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+  if (titleMatch?.[1]) {
+    return titleMatch[1].trim();
+  }
+  return "";
+}
+function tokenOverlap(tokensA, tokensB) {
+  if (tokensA.length === 0 || tokensB.length === 0) return 0;
+  const setB = new Set(tokensB);
+  let matches = 0;
+  for (const token of tokensA) {
+    if (setB.has(token)) matches++;
+  }
+  return matches / tokensA.length;
+}
+function getUrlPathTokens(url) {
+  try {
+    const urlObj = new URL(url);
+    const segments = urlObj.pathname.split("/").filter((s) => s.length > 0);
+    const tokens = [];
+    for (const segment of segments) {
+      tokens.push(...tokenize(segment.replace(/-/g, " ")));
+    }
+    return tokens;
+  } catch {
+    return [];
+  }
+}
+function computePageRelevance(page, queryTokens, searxngScore) {
+  const titleTokens = tokenize(page.title);
+  const titleMatch = tokenOverlap(queryTokens, titleTokens);
+  const urlTokens = getUrlPathTokens(page.url);
+  const urlMatch = tokenOverlap(queryTokens, urlTokens);
+  const bestExcerptScore = page.excerpts.length > 0 ? Math.max(...page.excerpts.map((e) => e.score)) : 0;
+  const normalizedSearxng = Math.min(searxngScore / 10, 1);
+  return PAGE_RELEVANCE_WEIGHTS.titleMatch * titleMatch + PAGE_RELEVANCE_WEIGHTS.urlMatch * urlMatch + PAGE_RELEVANCE_WEIGHTS.excerptScore * bestExcerptScore + PAGE_RELEVANCE_WEIGHTS.searxngScore * normalizedSearxng;
+}
+function processPage(url, html, query, charBudget, searchResult) {
+  try {
+    const result = extractExcerpts(html, query, {
+      ...SEARCH_PIPELINE_CONFIG,
+      excerpts: {
+        charBudget,
+        maxExcerpts: 5
+        // More excerpts for comprehensive docs
+      }
+    });
+    const title = searchResult?.title ?? extractTitle(html) ?? url;
+    return {
+      extraction: {
+        url,
+        title,
+        excerpts: result.excerpts.map((e) => ({
+          text: e.text,
+          headingPath: e.headingPath,
+          score: e.score
+        })),
+        totalChars: result.totalChars
+      },
+      relevanceMetrics: result.relevanceMetrics
+    };
+  } catch (error) {
+    return {
+      extraction: {
+        url,
+        title: searchResult?.title ?? url,
+        excerpts: [],
+        totalChars: 0,
+        error: error instanceof Error ? error.message : "Unknown extraction error"
+      },
+      relevanceMetrics: void 0
+    };
+  }
+}
+function getShortUrl(url) {
+  try {
+    const parsed = new URL(url);
+    const domain = parsed.hostname.replace(/^www\./, "");
+    let path = parsed.pathname;
+    if (path.length > 40) {
+      path = path.slice(0, 37) + "...";
+    }
+    return domain + path;
+  } catch {
+    return url;
+  }
+}
+function formatPageStatus(diag) {
+  const shortUrl = getShortUrl(diag.url);
+  switch (diag.status) {
+    case "success":
+      return `+ [${shortUrl}] ${diag.excerptCount} excerpt(s), ${diag.charCount} chars`;
+    case "scrape_failed":
+      return `- [${shortUrl}] FAILED: ${diag.error ?? "Could not fetch page"}`;
+    case "no_content":
+      return `- [${shortUrl}] SKIPPED: No extractable content`;
+    case "not_relevant": {
+      const coverage = diag.metrics?.queryTermCoverage ?? 0;
+      const bm25 = diag.metrics?.maxBm25 ?? 0;
+      const coveragePct = (coverage * 100).toFixed(0);
+      return `- [${shortUrl}] SKIPPED: Not relevant (${coveragePct}% terms, score: ${bm25.toFixed(2)})`;
+    }
+    case "budget_exceeded":
+      return `~ [${shortUrl}] TRUNCATED: Output limit reached`;
+    case "blocked_js":
+      return `x [${shortUrl}] SKIPPED: JS-rendered site`;
+    default:
+      return `? [${shortUrl}] Unknown status`;
+  }
+}
+function generateSuggestions(diagnostics, queryTokens) {
+  const suggestions = [];
+  const successCount = diagnostics.filter((d) => d.status === "success").length;
+  const notRelevantPages = diagnostics.filter((d) => d.status === "not_relevant");
+  const noContentCount = diagnostics.filter((d) => d.status === "no_content").length;
+  const scrapeFailedCount = diagnostics.filter((d) => d.status === "scrape_failed").length;
+  const budgetExceededCount = diagnostics.filter((d) => d.status === "budget_exceeded").length;
+  const blockedJsCount = diagnostics.filter((d) => d.status === "blocked_js").length;
+  if (successCount === 0) {
+    suggestions.push("NO RESULTS EXTRACTED. Your query may be too vague or not match the content.");
+    suggestions.push("Try a more specific query with exact library names, function names, or error messages.");
+    suggestions.push(`Current query tokens: [${queryTokens.join(", ")}] - ensure these terms appear in documentation you're looking for.`);
+    if (blockedJsCount > 0) {
+      suggestions.push(`${blockedJsCount} result(s) from Stack Overflow/GitHub were skipped (JavaScript-rendered sites not supported).`);
+    }
+    return suggestions;
+  }
+  if (successCount < diagnostics.length / 2) {
+    suggestions.push(`Only ${successCount}/${diagnostics.length} pages had relevant content.`);
+  }
+  if (notRelevantPages.length >= 2) {
+    const avgCoverage = notRelevantPages.reduce((sum, p) => sum + (p.metrics?.queryTermCoverage ?? 0), 0) / notRelevantPages.length;
+    if (avgCoverage < 0.3) {
+      suggestions.push("Query terms not found on many pages. Try adding the exact library/framework name (e.g., 'React', 'Next.js', 'Express').");
+    } else {
+      suggestions.push("Pages contained some query terms but lacked focused content. Try adding method names or error codes.");
+    }
+  }
+  if (scrapeFailedCount >= 2) {
+    suggestions.push("Multiple pages failed to load. This is normal for paywalled or bot-protected sites.");
+  }
+  if (noContentCount >= 2) {
+    suggestions.push("Multiple pages had no extractable content. These may require login or have unusual page structures.");
+  }
+  if (budgetExceededCount > 0) {
+    suggestions.push(`${budgetExceededCount} additional page(s) had content but were omitted due to output size limits.`);
+  }
+  if (blockedJsCount > 0) {
+    suggestions.push(`${blockedJsCount} result(s) from Stack Overflow/GitHub were skipped (JavaScript-rendered).`);
+  }
+  return suggestions;
+}
+function formatResults(result, includeDiagnostics) {
+  const lines = [];
+  lines.push(`# Search Results for: "${result.query}"
+`);
+  lines.push(`Found ${result.successfulPages} of ${result.totalPages} pages with relevant content.
+`);
+  for (const page of result.pages) {
+    if (page.excerpts.length === 0) continue;
+    lines.push(`
+## ${page.title}`);
+    lines.push(`Source: ${page.url}
+`);
+    for (const excerpt of page.excerpts) {
+      if (excerpt.headingPath.length > 0) {
+        lines.push(`### ${excerpt.headingPath.join(" > ")}
+`);
+      }
+      lines.push(excerpt.text);
+      lines.push("");
+    }
+  }
+  const noResults = result.pages.every((p) => p.excerpts.length === 0);
+  if (noResults) {
+    lines.push("\nNo relevant content was extracted from any search result.\n");
+  }
+  if (includeDiagnostics || noResults) {
+    lines.push("\n---\n");
+    lines.push("## Search Diagnostics\n");
+    lines.push("**Query Analysis:**");
+    lines.push(`- Original query: "${result.query}"`);
+    lines.push(`- Tokenized to: [${result.queryTokens.join(", ")}]`);
+    lines.push(`- ${result.queryTokens.length} search term(s) used for matching
+`);
+    lines.push("**Page-by-Page Results:**");
+    for (const diag of result.diagnostics) {
+      lines.push(`- ${formatPageStatus(diag)}`);
+    }
+    const suggestions = generateSuggestions(result.diagnostics, result.queryTokens);
+    if (suggestions.length > 0) {
+      lines.push("\n**Suggestions:**");
+      for (const suggestion of suggestions) {
+        lines.push(suggestion);
+      }
+    }
+  }
+  return lines.join("\n");
+}
+async function search(query, config = {}) {
+  const debug = config.debug ?? false;
+  const includeDiagnostics = config.diagnostics ?? false;
+  const { searchQuery, extractionQuery } = parseSearchOperators(query);
+  const cfg = {
+    searxngUrl: config.searxngUrl ?? DEFAULT_CONFIG7.searxngUrl,
+    maxResults: config.maxResults ?? DEFAULT_CONFIG7.maxResults,
+    timeout: config.timeout ?? DEFAULT_CONFIG7.timeout,
+    perPageCharBudget: config.perPageCharBudget ?? DEFAULT_CONFIG7.perPageCharBudget,
+    totalCharBudget: config.totalCharBudget ?? DEFAULT_CONFIG7.totalCharBudget
+  };
+  const requestMultiplier = 2;
+  let searchResults;
+  try {
+    searchResults = await logger.timeAsync("MCP: SearXNG search", async () => searchSearxng(searchQuery, {
+      baseUrl: cfg.searxngUrl,
+      maxResults: cfg.maxResults * requestMultiplier,
+      timeout: cfg.timeout
+    }));
+  } catch (error) {
+    return `Error searching SearXNG: ${error instanceof Error ? error.message : "Unknown error"}`;
+  }
+  if (searchResults.length === 0) {
+    return `No search results found for: "${query}"`;
+  }
+  const deduplicatedResults = deduplicateUrls(searchResults);
+  logger.debug(`Deduplicated ${searchResults.length} URLs to ${deduplicatedResults.length}`, debug);
+  deduplicatedResults.sort((a, b) => {
+    const scoreDiff = b.score - a.score;
+    if (scoreDiff !== 0) return scoreDiff;
+    return a.url.localeCompare(b.url);
+  });
+  const blockedResults = [];
+  const scrapableResults = [];
+  for (const result2 of deduplicatedResults) {
+    if (isBlockedDomain(result2.url)) {
+      blockedResults.push(result2);
+    } else {
+      scrapableResults.push(result2);
+    }
+  }
+  const resultsToScrape = scrapableResults.slice(0, cfg.maxResults);
+  logger.debug(`SearXNG returned ${searchResults.length} URLs, ${blockedResults.length} blocked, ${resultsToScrape.length} to scrape:`, debug);
+  for (let i = 0; i < resultsToScrape.length; i++) {
+    const r = resultsToScrape[i];
+    if (r) {
+      logger.debug(`  ${i + 1}. [score=${r.score.toFixed(2)}] ${r.url}`, debug);
+    }
+  }
+  if (blockedResults.length > 0) {
+    logger.debug(`Blocked domains:`, debug);
+    for (const r of blockedResults) {
+      logger.debug(`  - ${r.url}`, debug);
+    }
+  }
+  const urls = resultsToScrape.map((r) => r.url);
+  const scrapeResults = await logger.timeAsync("MCP: Scrape pages", async () => scrapeUrls(urls, { timeout: cfg.timeout }));
+  const searchResultMap = /* @__PURE__ */ new Map();
+  for (const sr of resultsToScrape) {
+    searchResultMap.set(sr.url, sr);
+  }
+  const diagnostics = [];
+  const blockedToShow = blockedResults.slice(0, 3);
+  for (const blocked of blockedToShow) {
+    diagnostics.push({
+      url: blocked.url,
+      title: blocked.title,
+      status: "blocked_js"
+    });
+  }
+  const pageExtractions = [];
+  const extractionStart = performance.now();
+  for (const scrape of scrapeResults) {
+    const searchResultInfo = searchResultMap.get(scrape.url);
+    const title = searchResultInfo?.title ?? scrape.url;
+    if (scrape.html === null) {
+      const diagEntry = {
+        url: scrape.url,
+        title,
+        status: "scrape_failed"
+      };
+      if (scrape.error !== void 0) {
+        diagEntry.error = scrape.error;
+      }
+      diagnostics.push(diagEntry);
+      const extraction = {
+        url: scrape.url,
+        title,
+        excerpts: [],
+        totalChars: 0
+      };
+      if (scrape.error !== void 0) {
+        extraction.error = scrape.error;
+      }
+      pageExtractions.push(extraction);
+      continue;
+    }
+    const result2 = processPage(
+      scrape.url,
+      scrape.html,
+      extractionQuery,
+      cfg.perPageCharBudget,
+      searchResultInfo
+    );
+    pageExtractions.push(result2.extraction);
+    if (result2.extraction.excerpts.length === 0) {
+      const metrics = result2.relevanceMetrics;
+      let status;
+      if (metrics && metrics.hasRelevantResults === false) {
+        status = "not_relevant";
+      } else if (metrics && metrics.sentenceCount === 0) {
+        status = "no_content";
+      } else {
+        status = "no_content";
+      }
+      const diagEntry = {
+        url: scrape.url,
+        title: result2.extraction.title,
+        status
+      };
+      if (metrics) {
+        diagEntry.metrics = {
+          sentenceCount: metrics.sentenceCount,
+          queryTermCoverage: metrics.queryTermCoverage,
+          maxBm25: metrics.maxBm25,
+          maxCooccurrence: metrics.maxCooccurrence
+        };
+      }
+      diagnostics.push(diagEntry);
+    } else {
+      diagnostics.push({
+        url: scrape.url,
+        title: result2.extraction.title,
+        status: "success",
+        excerptCount: result2.extraction.excerpts.length,
+        charCount: result2.extraction.totalChars
+      });
+    }
+  }
+  const extractionDuration = performance.now() - extractionStart;
+  logger.recordTiming(`MCP: Extract excerpts (${scrapeResults.length} pages)`, extractionDuration);
+  const rankingStart = performance.now();
+  const queryTokens = tokenize(extractionQuery);
+  const successfulPages = pageExtractions.filter((p) => p.excerpts.length > 0);
+  const pagesWithRelevance = successfulPages.map((page) => {
+    const searxngScore = searchResultMap.get(page.url)?.score ?? 0;
+    const relevance = computePageRelevance(page, queryTokens, searxngScore);
+    return { page, relevance };
+  });
+  pagesWithRelevance.sort((a, b) => {
+    const d = b.relevance - a.relevance;
+    if (d !== 0) return d;
+    return a.page.url.localeCompare(b.page.url);
+  });
+  let totalChars = 0;
+  const budgetedPages = [];
+  const includedUrls = /* @__PURE__ */ new Set();
+  for (const { page } of pagesWithRelevance) {
+    if (totalChars + page.totalChars > cfg.totalCharBudget) {
+      const remainingBudget = cfg.totalCharBudget - totalChars;
+      if (remainingBudget > 200) {
+        let pageChars = 0;
+        const trimmedExcerpts = [];
+        for (const excerpt of page.excerpts) {
+          const excerptChars = excerpt.text.length;
+          if (pageChars + excerptChars <= remainingBudget) {
+            trimmedExcerpts.push(excerpt);
+            pageChars += excerptChars;
+          }
+        }
+        if (trimmedExcerpts.length > 0) {
+          budgetedPages.push({
+            ...page,
+            excerpts: trimmedExcerpts,
+            totalChars: pageChars
+          });
+          includedUrls.add(page.url);
+          totalChars += pageChars;
+        }
+      }
+      break;
+    }
+    budgetedPages.push(page);
+    includedUrls.add(page.url);
+    totalChars += page.totalChars;
+  }
+  for (const { page } of pagesWithRelevance) {
+    if (!includedUrls.has(page.url)) {
+      const diagIndex = diagnostics.findIndex((d) => d.url === page.url);
+      if (diagIndex !== -1) {
+        diagnostics[diagIndex] = {
+          url: page.url,
+          title: page.title,
+          status: "budget_exceeded",
+          excerptCount: page.excerpts.length,
+          charCount: page.totalChars
+        };
+      }
+    }
+  }
+  logger.recordTiming("MCP: Rank and budget pages", performance.now() - rankingStart);
+  const result = {
+    query,
+    pages: budgetedPages,
+    totalPages: resultsToScrape.length,
+    // Pages we actually attempted to scrape
+    successfulPages: budgetedPages.length,
+    totalChars,
+    diagnostics,
+    queryTokens
+  };
+  const formatted = logger.time("MCP: Format results", () => formatResults(result, includeDiagnostics));
+  logger.printTimings();
+  return formatted;
+}
+
+export {
+  tokenize,
+  formatExcerpts,
+  logger_default,
+  extractExcerpts,
+  search
+};