searchsocket 0.3.2 → 0.4.0

package/dist/index.cjs

@@ -5025,32 +5025,32 @@ var require_URL = __commonJS({
           else
             return basepath.substring(0, lastslash + 1) + refpath;
         }
-        function remove_dot_segments(path14) {
-          if (!path14) return path14;
+        function remove_dot_segments(path15) {
+          if (!path15) return path15;
           var output = "";
-          while (path14.length > 0) {
-            if (path14 === "." || path14 === "..") {
-              path14 = "";
+          while (path15.length > 0) {
+            if (path15 === "." || path15 === "..") {
+              path15 = "";
               break;
             }
-            var twochars = path14.substring(0, 2);
-            var threechars = path14.substring(0, 3);
-            var fourchars = path14.substring(0, 4);
+            var twochars = path15.substring(0, 2);
+            var threechars = path15.substring(0, 3);
+            var fourchars = path15.substring(0, 4);
             if (threechars === "../") {
-              path14 = path14.substring(3);
+              path15 = path15.substring(3);
             } else if (twochars === "./") {
-              path14 = path14.substring(2);
+              path15 = path15.substring(2);
             } else if (threechars === "/./") {
-              path14 = "/" + path14.substring(3);
-            } else if (twochars === "/." && path14.length === 2) {
-              path14 = "/";
-            } else if (fourchars === "/../" || threechars === "/.." && path14.length === 3) {
-              path14 = "/" + path14.substring(4);
+              path15 = "/" + path15.substring(3);
+            } else if (twochars === "/." && path15.length === 2) {
+              path15 = "/";
+            } else if (fourchars === "/../" || threechars === "/.." && path15.length === 3) {
+              path15 = "/" + path15.substring(4);
               output = output.replace(/\/?[^\/]*$/, "");
             } else {
-              var segment = path14.match(/(\/?([^\/]*))/)[0];
+              var segment = path15.match(/(\/?([^\/]*))/)[0];
               output += segment;
-              path14 = path14.substring(segment.length);
+              path15 = path15.substring(segment.length);
             }
           }
           return output;
@@ -16614,6 +16614,8 @@ var searchSocketConfigSchema = zod.z.object({
     envVar: zod.z.string().min(1).optional(),
     sanitize: zod.z.boolean().optional()
   }).optional(),
+  exclude: zod.z.array(zod.z.string()).optional(),
+  respectRobotsTxt: zod.z.boolean().optional(),
   source: zod.z.object({
     mode: zod.z.enum(["static-output", "crawl", "content-files", "build"]).optional(),
     staticOutputDir: zod.z.string().min(1).optional(),
@@ -16744,6 +16746,8 @@ function createDefaultConfig(projectId) {
       envVar: "SEARCHSOCKET_SCOPE",
       sanitize: true
     },
+    exclude: [],
+    respectRobotsTxt: true,
     source: {
       mode: "static-output",
       staticOutputDir: "build",
@@ -16774,7 +16778,7 @@ function createDefaultConfig(projectId) {
     },
     embeddings: {
       provider: "jina",
-      model: "jina-embeddings-v3",
+      model: "jina-embeddings-v5-text-small",
       apiKeyEnv: "JINA_API_KEY",
       batchSize: 64,
       concurrency: 4
@@ -16787,9 +16791,9 @@ function createDefaultConfig(projectId) {
       }
     },
     rerank: {
-      enabled: false,
+      enabled: true,
       topN: 20,
-      model: "jina-reranker-v2-base-multilingual"
+      model: "jina-reranker-v3"
     },
     ranking: {
       enableIncomingLinkBoost: true,
@@ -16908,6 +16912,8 @@ ${issues}`
       ...defaults.scope,
       ...parsed.scope
     },
+    exclude: parsed.exclude ?? defaults.exclude,
+    respectRobotsTxt: parsed.respectRobotsTxt ?? defaults.respectRobotsTxt,
     source: {
       ...defaults.source,
       ...parsed.source,
@@ -17273,7 +17279,7 @@ var JinaReranker = class {
   constructor(options) {
     this.apiKey = options.apiKey;
     this.model = options.model;
-    this.maxRetries = options.maxRetries ?? 4;
+    this.maxRetries = options.maxRetries ?? 2;
   }
   async rerank(query, candidates, topN) {
     if (candidates.length === 0) {
@@ -17283,7 +17289,8 @@ var JinaReranker = class {
       model: this.model,
       query,
       documents: candidates.map((candidate) => candidate.text),
-      top_n: topN ?? candidates.length
+      top_n: topN ?? candidates.length,
+      return_documents: false
     };
     let attempt = 0;
     while (attempt <= this.maxRetries) {
@@ -19048,6 +19055,17 @@ function extractFromHtml(url, html, config) {
   if ($(`[${config.extract.noindexAttr}]`).length > 0) {
     return null;
   }
+  const weightRaw = $("meta[name='searchsocket-weight']").attr("content")?.trim();
+  let weight;
+  if (weightRaw !== void 0) {
+    const parsed = Number(weightRaw);
+    if (Number.isFinite(parsed) && parsed >= 0) {
+      weight = parsed;
+    }
+  }
+  if (weight === 0) {
+    return null;
+  }
   const description = $("meta[name='description']").attr("content")?.trim() || $("meta[property='og:description']").attr("content")?.trim() || void 0;
   const keywordsRaw = $("meta[name='keywords']").attr("content")?.trim();
   const keywords = keywordsRaw ? keywordsRaw.split(",").map((k) => k.trim()).filter(Boolean) : void 0;
@@ -19103,7 +19121,8 @@ function extractFromHtml(url, html, config) {
     noindex: false,
     tags,
     description,
-    keywords
+    keywords,
+    weight
   };
 }
 function extractFromMarkdown(url, markdown, title) {
@@ -19116,6 +19135,14 @@ function extractFromMarkdown(url, markdown, title) {
   if (frontmatter.noindex === true || searchsocketMeta?.noindex === true) {
     return null;
   }
+  let mdWeight;
+  const rawWeight = searchsocketMeta?.weight ?? frontmatter.searchsocketWeight;
+  if (typeof rawWeight === "number" && Number.isFinite(rawWeight) && rawWeight >= 0) {
+    mdWeight = rawWeight;
+  }
+  if (mdWeight === 0) {
+    return null;
+  }
   const content = parsed.content;
   const normalized = normalizeMarkdown(content);
   if (!normalizeText(normalized)) {
@@ -19138,7 +19165,8 @@ function extractFromMarkdown(url, markdown, title) {
     noindex: false,
     tags: normalizeUrlPath(url).split("/").filter(Boolean).slice(0, 1),
     description: fmDescription,
-    keywords: fmKeywords
+    keywords: fmKeywords,
+    weight: mdWeight
   };
 }
 function yamlString(value) {
@@ -19334,6 +19362,38 @@ var Logger = class {
 `);
   }
 };
+
+// src/utils/pattern.ts
+function matchUrlPattern(url, pattern) {
+  const norm = (p) => p !== "/" && p.endsWith("/") ? p.slice(0, -1) : p;
+  const normalizedUrl = norm(url);
+  const normalizedPattern = norm(pattern);
+  if (normalizedPattern.endsWith("/**")) {
+    const prefix = normalizedPattern.slice(0, -3);
+    if (prefix === "") {
+      return true;
+    }
+    return normalizedUrl === prefix || normalizedUrl.startsWith(prefix + "/");
+  }
+  if (normalizedPattern.endsWith("/*")) {
+    const prefix = normalizedPattern.slice(0, -2);
+    if (prefix === "") {
+      return normalizedUrl !== "/" && !normalizedUrl.slice(1).includes("/");
+    }
+    if (!normalizedUrl.startsWith(prefix + "/")) return false;
+    const rest = normalizedUrl.slice(prefix.length + 1);
+    return rest.length > 0 && !rest.includes("/");
+  }
+  return normalizedUrl === normalizedPattern;
+}
+function matchUrlPatterns(url, patterns) {
+  for (const pattern of patterns) {
+    if (matchUrlPattern(url, pattern)) return true;
+  }
+  return false;
+}
+
+// src/indexing/sources/build/manifest-parser.ts
 function routeIdToFile(routeId) {
   if (routeId === "/") {
     return "src/routes/+page.svelte";
@@ -19407,15 +19467,7 @@ function expandDynamicUrl(url, value) {
   return url.replace(/\[\[?\.\.\.[^\]]+\]?\]|\[\[[^\]]+\]\]|\[[^\]]+\]/g, value);
 }
 function isExcluded(url, patterns) {
-  for (const pattern of patterns) {
-    if (pattern.endsWith("/*")) {
-      const prefix = pattern.slice(0, -1);
-      if (url.startsWith(prefix) || url === prefix.slice(0, -1)) return true;
-    } else if (url === pattern) {
-      return true;
-    }
-  }
-  return false;
+  return matchUrlPatterns(url, patterns);
 }
 function findFreePort() {
   return new Promise((resolve, reject) => {
@@ -19831,6 +19883,158 @@ async function loadStaticOutputPages(cwd, config, maxPages) {
   }
   return pages;
 }
+function parseRobotsTxt(content, userAgent = "Searchsocket") {
+  const lines = content.split(/\r?\n/);
+  const agentGroups = /* @__PURE__ */ new Map();
+  let currentAgents = [];
+  for (const rawLine of lines) {
+    const line = rawLine.replace(/#.*$/, "").trim();
+    if (!line) continue;
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) continue;
+    const directive = line.slice(0, colonIdx).trim().toLowerCase();
+    const value = line.slice(colonIdx + 1).trim();
+    if (directive === "user-agent") {
+      const agentName = value.toLowerCase();
+      currentAgents.push(agentName);
+      if (!agentGroups.has(agentName)) {
+        agentGroups.set(agentName, { disallow: [], allow: [] });
+      }
+    } else if (directive === "disallow" && value && currentAgents.length > 0) {
+      for (const agent of currentAgents) {
+        agentGroups.get(agent).disallow.push(value);
+      }
+    } else if (directive === "allow" && value && currentAgents.length > 0) {
+      for (const agent of currentAgents) {
+        agentGroups.get(agent).allow.push(value);
+      }
+    } else if (directive !== "disallow" && directive !== "allow") {
+      currentAgents = [];
+    }
+  }
+  const specific = agentGroups.get(userAgent.toLowerCase());
+  if (specific && (specific.disallow.length > 0 || specific.allow.length > 0)) {
+    return specific;
+  }
+  return agentGroups.get("*") ?? { disallow: [], allow: [] };
+}
+function isBlockedByRobots(urlPath, rules3) {
+  let longestDisallow = "";
+  for (const pattern of rules3.disallow) {
+    if (urlPath.startsWith(pattern) && pattern.length > longestDisallow.length) {
+      longestDisallow = pattern;
+    }
+  }
+  if (!longestDisallow) return false;
+  let longestAllow = "";
+  for (const pattern of rules3.allow) {
+    if (urlPath.startsWith(pattern) && pattern.length > longestAllow.length) {
+      longestAllow = pattern;
+    }
+  }
+  return longestAllow.length < longestDisallow.length;
+}
+async function loadRobotsTxtFromDir(dir) {
+  try {
+    const content = await fs4__default.default.readFile(path__default.default.join(dir, "robots.txt"), "utf8");
+    return parseRobotsTxt(content);
+  } catch {
+    return null;
+  }
+}
+async function fetchRobotsTxt(baseUrl) {
+  try {
+    const url = new URL("/robots.txt", baseUrl).href;
+    const response = await fetch(url);
+    if (!response.ok) return null;
+    const content = await response.text();
+    return parseRobotsTxt(content);
+  } catch {
+    return null;
+  }
+}
+
+// src/search/ranking.ts
+function nonNegativeOrZero(value) {
+  if (!Number.isFinite(value)) {
+    return 0;
+  }
+  return Math.max(0, value);
+}
+function rankHits(hits, config) {
+  return hits.map((hit) => {
+    let score = Number.isFinite(hit.score) ? hit.score : Number.NEGATIVE_INFINITY;
+    if (config.ranking.enableIncomingLinkBoost) {
+      const incomingBoost = Math.log(1 + nonNegativeOrZero(hit.metadata.incomingLinks));
+      score += incomingBoost * config.ranking.weights.incomingLinks;
+    }
+    if (config.ranking.enableDepthBoost) {
+      const depthBoost = 1 / (1 + nonNegativeOrZero(hit.metadata.depth));
+      score += depthBoost * config.ranking.weights.depth;
+    }
+    return {
+      hit,
+      finalScore: Number.isFinite(score) ? score : Number.NEGATIVE_INFINITY
+    };
+  }).sort((a, b) => {
+    const delta = b.finalScore - a.finalScore;
+    return Number.isNaN(delta) ? 0 : delta;
+  });
+}
+function findPageWeight(url, pageWeights) {
+  let bestPattern = "";
+  let bestWeight = 1;
+  for (const [pattern, weight] of Object.entries(pageWeights)) {
+    if (matchUrlPattern(url, pattern) && pattern.length > bestPattern.length) {
+      bestPattern = pattern;
+      bestWeight = weight;
+    }
+  }
+  return bestWeight;
+}
+function aggregateByPage(ranked, config) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const hit of ranked) {
+    const url = hit.hit.metadata.url;
+    const group = groups.get(url);
+    if (group) group.push(hit);
+    else groups.set(url, [hit]);
+  }
+  const { aggregationCap, aggregationDecay } = config.ranking;
+  const pages = [];
+  for (const [url, chunks] of groups) {
+    chunks.sort((a, b) => {
+      const delta = b.finalScore - a.finalScore;
+      return Number.isNaN(delta) ? 0 : delta;
+    });
+    const best = chunks[0];
+    const maxScore = Number.isFinite(best.finalScore) ? best.finalScore : Number.NEGATIVE_INFINITY;
+    const topChunks = chunks.slice(0, aggregationCap);
+    let aggregationBonus = 0;
+    for (let i = 1; i < topChunks.length; i++) {
+      const chunkScore = Number.isFinite(topChunks[i].finalScore) ? topChunks[i].finalScore : 0;
+      aggregationBonus += chunkScore * Math.pow(aggregationDecay, i);
+    }
+    let pageScore = maxScore + aggregationBonus * config.ranking.weights.aggregation;
+    const pageWeight = findPageWeight(url, config.ranking.pageWeights);
+    if (pageWeight === 0) continue;
+    if (pageWeight !== 1) {
+      pageScore *= pageWeight;
+    }
+    pages.push({
+      url,
+      title: best.hit.metadata.title,
+      routeFile: best.hit.metadata.routeFile,
+      pageScore: Number.isFinite(pageScore) ? pageScore : Number.NEGATIVE_INFINITY,
+      bestChunk: best,
+      matchingChunks: chunks
+    });
+  }
+  return pages.sort((a, b) => {
+    const delta = b.pageScore - a.pageScore;
+    return Number.isNaN(delta) ? 0 : delta;
+  });
+}
 
 // src/utils/time.ts
 function nowIso() {
@@ -19842,9 +20046,10 @@ function hrTimeMs(start) {
 
 // src/indexing/pipeline.ts
 var EMBEDDING_PRICE_PER_1K_TOKENS_USD = {
-  "jina-embeddings-v3": 2e-5
+  "jina-embeddings-v3": 2e-5,
+  "jina-embeddings-v5-text-small": 5e-5
 };
-var DEFAULT_EMBEDDING_PRICE_PER_1K = 2e-5;
+var DEFAULT_EMBEDDING_PRICE_PER_1K = 5e-5;
 var IndexPipeline = class _IndexPipeline {
   cwd;
   config;
@@ -19922,6 +20127,53 @@ var IndexPipeline = class _IndexPipeline {
     }
     stageEnd("source", sourceStart);
     this.logger.info(`Loaded ${sourcePages.length} page${sourcePages.length === 1 ? "" : "s"} (${stageTimingsMs["source"]}ms)`);
+    const filterStart = stageStart();
+    let filteredSourcePages = sourcePages;
+    if (this.config.exclude.length > 0) {
+      const beforeExclude = filteredSourcePages.length;
+      filteredSourcePages = filteredSourcePages.filter((p) => {
+        const url = normalizeUrlPath(p.url);
+        if (matchUrlPatterns(url, this.config.exclude)) {
+          this.logger.debug(`Excluding ${url} (matched exclude pattern)`);
+          return false;
+        }
+        return true;
+      });
+      const excludedCount = beforeExclude - filteredSourcePages.length;
+      if (excludedCount > 0) {
+        this.logger.info(`Excluded ${excludedCount} page${excludedCount === 1 ? "" : "s"} by config exclude patterns`);
+      }
+    }
+    if (this.config.respectRobotsTxt) {
+      let robotsRules = null;
+      if (sourceMode === "static-output") {
+        robotsRules = await loadRobotsTxtFromDir(
+          path__default.default.resolve(this.cwd, this.config.source.staticOutputDir)
+        );
+      } else if (sourceMode === "build" && this.config.source.build) {
+        robotsRules = await loadRobotsTxtFromDir(
+          path__default.default.resolve(this.cwd, this.config.source.build.outputDir)
+        );
+      } else if (sourceMode === "crawl" && this.config.source.crawl) {
+        robotsRules = await fetchRobotsTxt(this.config.source.crawl.baseUrl);
+      }
+      if (robotsRules) {
+        const beforeRobots = filteredSourcePages.length;
+        filteredSourcePages = filteredSourcePages.filter((p) => {
+          const url = normalizeUrlPath(p.url);
+          if (isBlockedByRobots(url, robotsRules)) {
+            this.logger.debug(`Excluding ${url} (blocked by robots.txt)`);
+            return false;
+          }
+          return true;
+        });
+        const robotsExcluded = beforeRobots - filteredSourcePages.length;
+        if (robotsExcluded > 0) {
+          this.logger.info(`Excluded ${robotsExcluded} page${robotsExcluded === 1 ? "" : "s"} by robots.txt`);
+        }
+      }
+    }
+    stageEnd("filter", filterStart);
     const routeStart = stageStart();
     const routePatterns = await buildRoutePatterns(this.cwd);
     stageEnd("route_map", routeStart);
@@ -19929,7 +20181,7 @@ var IndexPipeline = class _IndexPipeline {
     const extractStart = stageStart();
     this.logger.info("Extracting content...");
     const extractedPages = [];
-    for (const sourcePage of sourcePages) {
+    for (const sourcePage of filteredSourcePages) {
       const extracted = sourcePage.html ? extractFromHtml(sourcePage.url, sourcePage.html, this.config) : extractFromMarkdown(sourcePage.url, sourcePage.markdown ?? "", sourcePage.title);
       if (!extracted) {
         this.logger.warn(
@@ -19955,16 +20207,29 @@ var IndexPipeline = class _IndexPipeline {
       seenUrls.add(page.url);
       uniquePages.push(page);
     }
+    const indexablePages = [];
+    for (const page of uniquePages) {
+      const effectiveWeight = page.weight ?? findPageWeight(page.url, this.config.ranking.pageWeights);
+      if (effectiveWeight === 0) {
+        this.logger.debug(`Excluding ${page.url} (zero weight)`);
+        continue;
+      }
+      indexablePages.push(page);
+    }
+    const zeroWeightCount = uniquePages.length - indexablePages.length;
+    if (zeroWeightCount > 0) {
+      this.logger.info(`Excluded ${zeroWeightCount} page${zeroWeightCount === 1 ? "" : "s"} with zero weight`);
+    }
     stageEnd("extract", extractStart);
-    const skippedPages = sourcePages.length - uniquePages.length;
-    this.logger.info(`Extracted ${uniquePages.length} page${uniquePages.length === 1 ? "" : "s"}${skippedPages > 0 ? ` (${skippedPages} skipped)` : ""} (${stageTimingsMs["extract"]}ms)`);
+    const skippedPages = filteredSourcePages.length - indexablePages.length;
+    this.logger.info(`Extracted ${indexablePages.length} page${indexablePages.length === 1 ? "" : "s"}${skippedPages > 0 ? ` (${skippedPages} skipped)` : ""} (${stageTimingsMs["extract"]}ms)`);
     const linkStart = stageStart();
-    const pageSet = new Set(uniquePages.map((page) => normalizeUrlPath(page.url)));
+    const pageSet = new Set(indexablePages.map((page) => normalizeUrlPath(page.url)));
     const incomingLinkCount = /* @__PURE__ */ new Map();
-    for (const page of uniquePages) {
+    for (const page of indexablePages) {
       incomingLinkCount.set(page.url, incomingLinkCount.get(page.url) ?? 0);
     }
-    for (const page of uniquePages) {
+    for (const page of indexablePages) {
       for (const outgoing of page.outgoingLinks) {
         if (!pageSet.has(outgoing)) {
           continue;
@@ -19988,7 +20253,7 @@ var IndexPipeline = class _IndexPipeline {
         });
       }
     }
-    for (const page of uniquePages) {
+    for (const page of indexablePages) {
       const routeMatch = precomputedRoutes.get(normalizeUrlPath(page.url)) ?? mapUrlToRoute(page.url, routePatterns);
       if (routeMatch.routeResolution === "best-effort") {
         if (this.config.source.strictRouteMapping) {
@@ -20205,100 +20470,6 @@ var IndexPipeline = class _IndexPipeline {
     };
   }
 };
-
-// src/search/ranking.ts
-function nonNegativeOrZero(value) {
-  if (!Number.isFinite(value)) {
-    return 0;
-  }
-  return Math.max(0, value);
-}
-function rankHits(hits, config) {
-  return hits.map((hit) => {
-    let score = Number.isFinite(hit.score) ? hit.score : Number.NEGATIVE_INFINITY;
-    if (config.ranking.enableIncomingLinkBoost) {
-      const incomingBoost = Math.log(1 + nonNegativeOrZero(hit.metadata.incomingLinks));
-      score += incomingBoost * config.ranking.weights.incomingLinks;
-    }
-    if (config.ranking.enableDepthBoost) {
-      const depthBoost = 1 / (1 + nonNegativeOrZero(hit.metadata.depth));
-      score += depthBoost * config.ranking.weights.depth;
-    }
-    return {
-      hit,
-      finalScore: Number.isFinite(score) ? score : Number.NEGATIVE_INFINITY
-    };
-  }).sort((a, b) => {
-    const delta = b.finalScore - a.finalScore;
-    return Number.isNaN(delta) ? 0 : delta;
-  });
-}
-function findPageWeight(url, pageWeights) {
-  const norm = (p) => p !== "/" && p.endsWith("/") ? p.slice(0, -1) : p;
-  const normalizedUrl = norm(url);
-  for (const [pattern, weight] of Object.entries(pageWeights)) {
-    if (norm(pattern) === normalizedUrl) {
-      return weight;
-    }
-  }
-  let bestPrefix = "";
-  let bestWeight = 1;
-  for (const [pattern, weight] of Object.entries(pageWeights)) {
-    const normalizedPattern = norm(pattern);
-    if (normalizedPattern === "/") continue;
-    const prefix = `${normalizedPattern}/`;
-    if (normalizedUrl.startsWith(prefix) && prefix.length > bestPrefix.length) {
-      bestPrefix = prefix;
-      bestWeight = weight;
-    }
-  }
-  return bestWeight;
-}
-function aggregateByPage(ranked, config) {
-  const groups = /* @__PURE__ */ new Map();
-  for (const hit of ranked) {
-    const url = hit.hit.metadata.url;
-    const group = groups.get(url);
-    if (group) group.push(hit);
-    else groups.set(url, [hit]);
-  }
-  const { aggregationCap, aggregationDecay } = config.ranking;
-  const pages = [];
-  for (const [url, chunks] of groups) {
-    chunks.sort((a, b) => {
-      const delta = b.finalScore - a.finalScore;
-      return Number.isNaN(delta) ? 0 : delta;
-    });
-    const best = chunks[0];
-    const maxScore = Number.isFinite(best.finalScore) ? best.finalScore : Number.NEGATIVE_INFINITY;
-    const topChunks = chunks.slice(0, aggregationCap);
-    let aggregationBonus = 0;
-    for (let i = 1; i < topChunks.length; i++) {
-      const chunkScore = Number.isFinite(topChunks[i].finalScore) ? topChunks[i].finalScore : 0;
-      aggregationBonus += chunkScore * Math.pow(aggregationDecay, i);
-    }
-    let pageScore = maxScore + aggregationBonus * config.ranking.weights.aggregation;
-    const pageWeight = findPageWeight(url, config.ranking.pageWeights);
-    if (pageWeight === 0) continue;
-    if (pageWeight !== 1) {
-      pageScore *= pageWeight;
-    }
-    pages.push({
-      url,
-      title: best.hit.metadata.title,
-      routeFile: best.hit.metadata.routeFile,
-      pageScore: Number.isFinite(pageScore) ? pageScore : Number.NEGATIVE_INFINITY,
-      bestChunk: best,
-      matchingChunks: chunks
-    });
-  }
-  return pages.sort((a, b) => {
-    const delta = b.pageScore - a.pageScore;
-    return Number.isNaN(delta) ? 0 : delta;
-  });
-}
-
-// src/search/engine.ts
 var requestSchema = zod.z.object({
   q: zod.z.string().trim().min(1),
   topK: zod.z.number().int().positive().max(100).optional(),
@@ -20306,7 +20477,8 @@ var requestSchema = zod.z.object({
   pathPrefix: zod.z.string().optional(),
   tags: zod.z.array(zod.z.string()).optional(),
   rerank: zod.z.boolean().optional(),
-  groupBy: zod.z.enum(["page", "chunk"]).optional()
+  groupBy: zod.z.enum(["page", "chunk"]).optional(),
+  stream: zod.z.boolean().optional()
 });
 var SearchEngine = class _SearchEngine {
   cwd;
@@ -20379,7 +20551,103 @@ var SearchEngine = class _SearchEngine {
       rerankMs = hrTimeMs(rerankStart);
       usedRerank = true;
     }
-    let results;
+    const results = this.buildResults(ordered, topK, groupByPage);
+    return {
+      q: input.q,
+      scope: resolvedScope.scopeName,
+      results,
+      meta: {
+        timingsMs: {
+          embed: Math.round(embedMs),
+          vector: Math.round(vectorMs),
+          rerank: Math.round(rerankMs),
+          total: Math.round(hrTimeMs(totalStart))
+        },
+        usedRerank,
+        modelId: this.config.embeddings.model
+      }
+    };
+  }
+  async *searchStreaming(request) {
+    const parsed = requestSchema.safeParse(request);
+    if (!parsed.success) {
+      throw new SearchSocketError("INVALID_REQUEST", parsed.error.issues[0]?.message ?? "Invalid request", 400);
+    }
+    const input = parsed.data;
+    const wantsRerank = Boolean(input.rerank);
+    if (!wantsRerank) {
+      const response = await this.search(request);
+      yield { phase: "initial", data: response };
+      return;
+    }
+    const totalStart = process.hrtime.bigint();
+    const resolvedScope = resolveScope(this.config, input.scope);
+    await this.assertModelCompatibility(resolvedScope);
+    const topK = input.topK ?? 10;
+    const groupByPage = (input.groupBy ?? "page") === "page";
+    const candidateK = groupByPage ? Math.max(topK * 10, 50) : Math.max(50, topK);
+    const embedStart = process.hrtime.bigint();
+    const queryEmbeddings = await this.embeddings.embedTexts([input.q], this.config.embeddings.model, "retrieval.query");
+    const queryVector = queryEmbeddings[0];
+    if (!queryVector || queryVector.length === 0 || queryVector.some((value) => !Number.isFinite(value))) {
+      throw new SearchSocketError("VECTOR_BACKEND_UNAVAILABLE", "Unable to create query embedding.");
+    }
+    const embedMs = hrTimeMs(embedStart);
+    const vectorStart = process.hrtime.bigint();
+    const hits = await this.vectorStore.query(
+      queryVector,
+      {
+        topK: candidateK,
+        pathPrefix: input.pathPrefix,
+        tags: input.tags
+      },
+      resolvedScope
+    );
+    const vectorMs = hrTimeMs(vectorStart);
+    const ranked = rankHits(hits, this.config);
+    const initialResults = this.buildResults(ranked, topK, groupByPage);
+    yield {
+      phase: "initial",
+      data: {
+        q: input.q,
+        scope: resolvedScope.scopeName,
+        results: initialResults,
+        meta: {
+          timingsMs: {
+            embed: Math.round(embedMs),
+            vector: Math.round(vectorMs),
+            rerank: 0,
+            total: Math.round(hrTimeMs(totalStart))
+          },
+          usedRerank: false,
+          modelId: this.config.embeddings.model
+        }
+      }
+    };
+    const rerankStart = process.hrtime.bigint();
+    const reranked = await this.rerankHits(input.q, ranked, topK);
+    const rerankMs = hrTimeMs(rerankStart);
+    const rerankedResults = this.buildResults(reranked, topK, groupByPage);
+    yield {
+      phase: "reranked",
+      data: {
+        q: input.q,
+        scope: resolvedScope.scopeName,
+        results: rerankedResults,
+        meta: {
+          timingsMs: {
+            embed: Math.round(embedMs),
+            vector: Math.round(vectorMs),
+            rerank: Math.round(rerankMs),
+            total: Math.round(hrTimeMs(totalStart))
+          },
+          usedRerank: true,
+          modelId: this.config.embeddings.model
+        }
+      }
+    };
+  }
+  buildResults(ordered, topK, groupByPage) {
     const minScore = this.config.ranking.minScore;
     if (groupByPage) {
       let pages = aggregateByPage(ordered, this.config);
@@ -20387,10 +20655,10 @@ var SearchEngine = class _SearchEngine {
         pages = pages.filter((p) => p.pageScore >= minScore);
       }
       const minRatio = this.config.ranking.minChunkScoreRatio;
-      results = pages.slice(0, topK).map((page) => {
+      return pages.slice(0, topK).map((page) => {
         const bestScore = page.bestChunk.finalScore;
-        const minScore2 = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
-        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minScore2).slice(0, 5);
+        const minChunkScore = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
+        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minChunkScore).slice(0, 5);
         return {
           url: page.url,
           title: page.title,
@@ -20407,10 +20675,11 @@ var SearchEngine = class _SearchEngine {
         };
       });
     } else {
+      let filtered = ordered;
       if (minScore > 0) {
-        ordered = ordered.filter((entry) => entry.finalScore >= minScore);
+        filtered = ordered.filter((entry) => entry.finalScore >= minScore);
       }
-      results = ordered.slice(0, topK).map(({ hit, finalScore }) => ({
+      return filtered.slice(0, topK).map(({ hit, finalScore }) => ({
         url: hit.metadata.url,
         title: hit.metadata.title,
         sectionTitle: hit.metadata.sectionTitle || void 0,
@@ -20419,21 +20688,6 @@ var SearchEngine = class _SearchEngine {
         routeFile: hit.metadata.routeFile
       }));
     }
-    return {
-      q: input.q,
-      scope: resolvedScope.scopeName,
-      results,
-      meta: {
-        timingsMs: {
-          embed: Math.round(embedMs),
-          vector: Math.round(vectorMs),
-          rerank: Math.round(rerankMs),
-          total: Math.round(hrTimeMs(totalStart))
-        },
-        usedRerank,
-        modelId: this.config.embeddings.model
-      }
-    };
   }
   async getPage(pathOrUrl, scope) {
     const resolvedScope = resolveScope(this.config, scope);
@@ -20505,6 +20759,7 @@ var SearchEngine = class _SearchEngine {
     const MAX_CHUNKS_PER_PAGE = 5;
     const MIN_CHUNKS_PER_PAGE = 1;
     const MIN_CHUNK_SCORE_RATIO = 0.5;
+    const MAX_DOC_CHARS = 2e3;
     const pageCandidates = [];
     for (const [url, chunks] of pageGroups) {
       const byScore = [...chunks].sort((a, b) => b.finalScore - a.finalScore);
@@ -20524,12 +20779,18 @@ var SearchEngine = class _SearchEngine {
       }
       const body = selected.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
       parts.push(body);
-      pageCandidates.push({ id: url, text: parts.join("\n\n") });
+      let text = parts.join("\n\n");
+      if (text.length > MAX_DOC_CHARS) {
+        text = text.slice(0, MAX_DOC_CHARS);
+      }
+      pageCandidates.push({ id: url, text });
     }
+    const maxCandidates = Math.max(topK, this.config.rerank.topN);
+    const cappedCandidates = pageCandidates.slice(0, maxCandidates);
     const reranked = await this.reranker.rerank(
       query,
-      pageCandidates,
-      Math.max(topK, this.config.rerank.topN)
+      cappedCandidates,
+      maxCandidates
     );
     const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {
@@ -20549,7 +20810,7 @@ var SearchEngine = class _SearchEngine {
     });
   }
 };
-function createServer(engine) {
+function createServer(engine, config) {
   const server = new mcp_js.McpServer({
     name: "searchsocket-mcp",
     version: "0.1.0"
@@ -20557,14 +20818,15 @@ function createServer(engine) {
   server.registerTool(
     "search",
     {
-      description: "Semantic site search. Returns url/title/snippet/score/routeFile for each match. Supports optional scope, pathPrefix, tags, and topK.",
+      description: "Semantic site search. Returns url/title/snippet/score/routeFile for each match. Supports optional scope, pathPrefix, tags, topK, and rerank. Enable rerank for better relevance on natural-language queries.",
       inputSchema: {
         query: zod.z.string().min(1),
         scope: zod.z.string().optional(),
         topK: zod.z.number().int().positive().max(100).optional(),
         pathPrefix: zod.z.string().optional(),
         tags: zod.z.array(zod.z.string()).optional(),
-        groupBy: zod.z.enum(["page", "chunk"]).optional()
+        groupBy: zod.z.enum(["page", "chunk"]).optional(),
+        rerank: zod.z.boolean().optional().describe("Enable reranking for better relevance (uses Jina Reranker). Defaults to true when rerank is enabled in config.")
       }
     },
     async (input) => {
@@ -20574,7 +20836,8 @@ function createServer(engine) {
         scope: input.scope,
         pathPrefix: input.pathPrefix,
         tags: input.tags,
-        groupBy: input.groupBy
+        groupBy: input.groupBy,
+        rerank: input.rerank ?? config.rerank.enabled
       });
       return {
         content: [
@@ -20700,10 +20963,10 @@ async function runMcpServer(options = {}) {
     config
   });
   if (resolvedTransport === "http") {
-    await startHttpServer(() => createServer(engine), config, options);
+    await startHttpServer(() => createServer(engine, config), config, options);
     return;
   }
-  const server = createServer(engine);
+  const server = createServer(engine, config);
   const stdioTransport = new stdio_js.StdioServerTransport();
   await server.connect(stdioTransport);
 }
@@ -20859,7 +21122,44 @@ function searchsocketHandle(options = {}) {
         throw new SearchSocketError("INVALID_REQUEST", "Malformed JSON request body", 400);
       }
       const engine = await getEngine();
-      const result = await engine.search(body);
+      const searchRequest = body;
+      if (searchRequest.stream && searchRequest.rerank) {
+        const encoder = new TextEncoder();
+        const stream = new ReadableStream({
+          async start(controller) {
+            try {
+              for await (const event2 of engine.searchStreaming(searchRequest)) {
+                const line = JSON.stringify(event2) + "\n";
+                controller.enqueue(encoder.encode(line));
+              }
+            } catch (streamError) {
+              const errorEvent = {
+                phase: "error",
+                data: {
+                  error: {
+                    code: streamError instanceof SearchSocketError ? streamError.code : "INTERNAL_ERROR",
+                    message: streamError instanceof Error ? streamError.message : "Unknown error"
+                  }
+                }
+              };
+              controller.enqueue(encoder.encode(JSON.stringify(errorEvent) + "\n"));
+            } finally {
+              controller.close();
+            }
+          }
+        });
+        return withCors(
+          new Response(stream, {
+            status: 200,
+            headers: {
+              "content-type": "application/x-ndjson"
+            }
+          }),
+          event.request,
+          config
+        );
+      }
+      const result = await engine.search(searchRequest);
       return withCors(
         new Response(JSON.stringify(result), {
           status: 200,
@@ -20972,7 +21272,7 @@ function searchsocketVitePlugin(options = {}) {
         });
         const stats = await pipeline.run({
           changedOnly: options.changedOnly ?? true,
-          force: options.force ?? false,
+          force: (options.force ?? false) || /^(1|true|yes)$/i.test(process.env.SEARCHSOCKET_FORCE_REINDEX ?? ""),
           dryRun: options.dryRun ?? false,
           scopeOverride: options.scope,
           verbose: options.verbose
@@ -20989,6 +21289,60 @@ function searchsocketVitePlugin(options = {}) {
   };
 }
 
+// src/merge.ts
+function mergeSearchResults(initial, reranked, options) {
+  const maxDisplacement = options?.maxDisplacement ?? 3;
+  const initialUrls = initial.results.map((r) => r.url);
+  const rerankedUrls = reranked.results.map((r) => r.url);
+  const initialPos = /* @__PURE__ */ new Map();
+  for (let i = 0; i < initialUrls.length; i++) {
+    initialPos.set(initialUrls[i], i);
+  }
+  const rerankedPos = /* @__PURE__ */ new Map();
+  for (let i = 0; i < rerankedUrls.length; i++) {
+    rerankedPos.set(rerankedUrls[i], i);
+  }
+  const displacements = [];
+  for (const url of initialUrls) {
+    const iPos = initialPos.get(url);
+    const rPos = rerankedPos.get(url);
+    const displacement = rPos !== void 0 ? Math.abs(iPos - rPos) : 0;
+    displacements.push({ url, displacement });
+  }
+  const totalResults = displacements.length;
+  if (totalResults === 0) {
+    return {
+      response: reranked,
+      usedRerankedOrder: true,
+      displacements
+    };
+  }
+  const hasLargeDisplacement = displacements.some((d) => d.displacement > maxDisplacement);
+  if (hasLargeDisplacement) {
+    return {
+      response: reranked,
+      usedRerankedOrder: true,
+      displacements
+    };
+  }
+  const rerankedScoreMap = /* @__PURE__ */ new Map();
+  for (const result of reranked.results) {
+    rerankedScoreMap.set(result.url, result.score);
+  }
+  const mergedResults = initial.results.map((result) => ({
+    ...result,
+    score: rerankedScoreMap.get(result.url) ?? result.score
+  }));
+  return {
+    response: {
+      ...reranked,
+      results: mergedResults
+    },
+    usedRerankedOrder: false,
+    displacements
+  };
+}
+
 // src/client.ts
 function createSearchClient(options = {}) {
   const endpoint = options.endpoint ?? "/api/search";
@@ -21016,6 +21370,72 @@ function createSearchClient(options = {}) {
         throw new Error(message);
       }
       return payload;
+    },
+    async streamSearch(request, onPhase) {
+      const response = await fetchImpl(endpoint, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json"
+        },
+        body: JSON.stringify(request)
+      });
+      if (!response.ok) {
+        let payload;
+        try {
+          payload = await response.json();
+        } catch {
+          throw new Error("Search failed");
+        }
+        const message = payload.error?.message ?? "Search failed";
+        throw new Error(message);
+      }
+      const contentType = response.headers.get("content-type") ?? "";
+      if (contentType.includes("application/json")) {
+        const data = await response.json();
+        onPhase({ phase: "initial", data });
+        return data;
+      }
+      if (!response.body) {
+        throw new Error("Response body is not readable");
+      }
+      const reader = response.body.getReader();
+      const decoder = new TextDecoder();
+      let buffer = "";
+      let lastResponse = null;
+      for (; ; ) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        let newlineIdx;
+        while ((newlineIdx = buffer.indexOf("\n")) !== -1) {
+          const line = buffer.slice(0, newlineIdx).trim();
+          buffer = buffer.slice(newlineIdx + 1);
+          if (line.length === 0) continue;
+          const event = JSON.parse(line);
+          if (event.phase === "error") {
+            const errData = event.data;
+            throw new Error(errData.error.message ?? "Streaming search error");
+          }
+          const searchEvent = event;
+          onPhase(searchEvent);
+          lastResponse = searchEvent.data;
+        }
+      }
+      const remaining = buffer.trim();
+      if (remaining.length > 0) {
+        const event = JSON.parse(remaining);
+        if (event.phase === "error") {
+          const errData = event.data;
+          throw new Error(errData.error.message ?? "Streaming search error");
+        }
+        const searchEvent = event;
+        onPhase(searchEvent);
+        lastResponse = searchEvent.data;
+      }
+      if (!lastResponse) {
+        throw new Error("No search results received");
+      }
+      return lastResponse;
     }
   };
 }
@@ -21042,6 +21462,7 @@ exports.isServerless = isServerless;
 exports.loadConfig = loadConfig;
 exports.mergeConfig = mergeConfig;
 exports.mergeConfigServerless = mergeConfigServerless;
+exports.mergeSearchResults = mergeSearchResults;
 exports.resolveScope = resolveScope;
 exports.runMcpServer = runMcpServer;
 exports.searchsocketHandle = searchsocketHandle;