searchsocket 0.3.2 → 0.4.0

package/dist/cli.js

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 
 // src/cli.ts
-import fs9 from "fs";
+import fs10 from "fs";
 import fsp from "fs/promises";
-import path13 from "path";
+import path14 from "path";
 import { execSync as execSync2 } from "child_process";
 import { config as dotenvConfig } from "dotenv";
 import chokidar from "chokidar";
@@ -12,7 +12,7 @@ import { Command } from "commander";
 // package.json
 var package_default = {
   name: "searchsocket",
-  version: "0.3.2",
+  version: "0.4.0",
   description: "Semantic site search and MCP retrieval for SvelteKit static sites",
   license: "MIT",
   author: "Greg Priday <greg@siteorigin.com>",
@@ -115,6 +115,8 @@ var searchSocketConfigSchema = z.object({
     envVar: z.string().min(1).optional(),
     sanitize: z.boolean().optional()
   }).optional(),
+  exclude: z.array(z.string()).optional(),
+  respectRobotsTxt: z.boolean().optional(),
   source: z.object({
     mode: z.enum(["static-output", "crawl", "content-files", "build"]).optional(),
     staticOutputDir: z.string().min(1).optional(),
@@ -245,6 +247,8 @@ function createDefaultConfig(projectId) {
       envVar: "SEARCHSOCKET_SCOPE",
       sanitize: true
     },
+    exclude: [],
+    respectRobotsTxt: true,
     source: {
       mode: "static-output",
       staticOutputDir: "build",
@@ -275,7 +279,7 @@ function createDefaultConfig(projectId) {
     },
     embeddings: {
       provider: "jina",
-      model: "jina-embeddings-v3",
+      model: "jina-embeddings-v5-text-small",
       apiKeyEnv: "JINA_API_KEY",
       batchSize: 64,
       concurrency: 4
@@ -288,9 +292,9 @@ function createDefaultConfig(projectId) {
       }
     },
     rerank: {
-      enabled: false,
+      enabled: true,
       topN: 20,
-      model: "jina-reranker-v2-base-multilingual"
+      model: "jina-reranker-v3"
     },
     ranking: {
       enableIncomingLinkBoost: true,
@@ -393,6 +397,8 @@ ${issues}`
       ...defaults.scope,
       ...parsed.scope
     },
+    exclude: parsed.exclude ?? defaults.exclude,
+    respectRobotsTxt: parsed.respectRobotsTxt ?? defaults.respectRobotsTxt,
     source: {
       ...defaults.source,
       ...parsed.source,
@@ -829,7 +835,7 @@ function createEmbeddingsProvider(config) {
 }
 
 // src/indexing/pipeline.ts
-import path11 from "path";
+import path12 from "path";
 
 // src/vector/factory.ts
 import fs3 from "fs";
@@ -1710,6 +1716,17 @@ function extractFromHtml(url, html, config) {
   if ($(`[${config.extract.noindexAttr}]`).length > 0) {
     return null;
   }
+  const weightRaw = $("meta[name='searchsocket-weight']").attr("content")?.trim();
+  let weight;
+  if (weightRaw !== void 0) {
+    const parsed = Number(weightRaw);
+    if (Number.isFinite(parsed) && parsed >= 0) {
+      weight = parsed;
+    }
+  }
+  if (weight === 0) {
+    return null;
+  }
   const description = $("meta[name='description']").attr("content")?.trim() || $("meta[property='og:description']").attr("content")?.trim() || void 0;
   const keywordsRaw = $("meta[name='keywords']").attr("content")?.trim();
   const keywords = keywordsRaw ? keywordsRaw.split(",").map((k) => k.trim()).filter(Boolean) : void 0;
@@ -1765,7 +1782,8 @@ function extractFromHtml(url, html, config) {
     noindex: false,
     tags,
     description,
-    keywords
+    keywords,
+    weight
   };
 }
 function extractFromMarkdown(url, markdown, title) {
@@ -1778,6 +1796,14 @@ function extractFromMarkdown(url, markdown, title) {
   if (frontmatter.noindex === true || searchsocketMeta?.noindex === true) {
     return null;
   }
+  let mdWeight;
+  const rawWeight = searchsocketMeta?.weight ?? frontmatter.searchsocketWeight;
+  if (typeof rawWeight === "number" && Number.isFinite(rawWeight) && rawWeight >= 0) {
+    mdWeight = rawWeight;
+  }
+  if (mdWeight === 0) {
+    return null;
+  }
   const content = parsed.content;
   const normalized = normalizeMarkdown(content);
   if (!normalizeText(normalized)) {
@@ -1800,7 +1826,8 @@ function extractFromMarkdown(url, markdown, title) {
     noindex: false,
     tags: normalizeUrlPath(url).split("/").filter(Boolean).slice(0, 1),
     description: fmDescription,
-    keywords: fmKeywords
+    keywords: fmKeywords,
+    weight: mdWeight
   };
 }
 
@@ -1937,6 +1964,38 @@ import pLimit2 from "p-limit";
 // src/indexing/sources/build/manifest-parser.ts
 import fs5 from "fs/promises";
 import path7 from "path";
+
+// src/utils/pattern.ts
+function matchUrlPattern(url, pattern) {
+  const norm = (p) => p !== "/" && p.endsWith("/") ? p.slice(0, -1) : p;
+  const normalizedUrl = norm(url);
+  const normalizedPattern = norm(pattern);
+  if (normalizedPattern.endsWith("/**")) {
+    const prefix = normalizedPattern.slice(0, -3);
+    if (prefix === "") {
+      return true;
+    }
+    return normalizedUrl === prefix || normalizedUrl.startsWith(prefix + "/");
+  }
+  if (normalizedPattern.endsWith("/*")) {
+    const prefix = normalizedPattern.slice(0, -2);
+    if (prefix === "") {
+      return normalizedUrl !== "/" && !normalizedUrl.slice(1).includes("/");
+    }
+    if (!normalizedUrl.startsWith(prefix + "/")) return false;
+    const rest = normalizedUrl.slice(prefix.length + 1);
+    return rest.length > 0 && !rest.includes("/");
+  }
+  return normalizedUrl === normalizedPattern;
+}
+function matchUrlPatterns(url, patterns) {
+  for (const pattern of patterns) {
+    if (matchUrlPattern(url, pattern)) return true;
+  }
+  return false;
+}
+
+// src/indexing/sources/build/manifest-parser.ts
 function routeIdToFile(routeId) {
   if (routeId === "/") {
     return "src/routes/+page.svelte";
@@ -2010,15 +2069,7 @@ function expandDynamicUrl(url, value) {
   return url.replace(/\[\[?\.\.\.[^\]]+\]?\]|\[\[[^\]]+\]\]|\[[^\]]+\]/g, value);
 }
 function isExcluded(url, patterns) {
-  for (const pattern of patterns) {
-    if (pattern.endsWith("/*")) {
-      const prefix = pattern.slice(0, -1);
-      if (url.startsWith(prefix) || url === prefix.slice(0, -1)) return true;
-    } else if (url === pattern) {
-      return true;
-    }
-  }
-  return false;
+  return matchUrlPatterns(url, patterns);
 }
 
 // src/indexing/sources/build/preview-server.ts
@@ -2456,6 +2507,162 @@ async function loadStaticOutputPages(cwd, config, maxPages) {
   return pages;
 }
 
+// src/indexing/robots.ts
+import fs9 from "fs/promises";
+import path11 from "path";
+function parseRobotsTxt(content, userAgent = "Searchsocket") {
+  const lines = content.split(/\r?\n/);
+  const agentGroups = /* @__PURE__ */ new Map();
+  let currentAgents = [];
+  for (const rawLine of lines) {
+    const line = rawLine.replace(/#.*$/, "").trim();
+    if (!line) continue;
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) continue;
+    const directive = line.slice(0, colonIdx).trim().toLowerCase();
+    const value = line.slice(colonIdx + 1).trim();
+    if (directive === "user-agent") {
+      const agentName = value.toLowerCase();
+      currentAgents.push(agentName);
+      if (!agentGroups.has(agentName)) {
+        agentGroups.set(agentName, { disallow: [], allow: [] });
+      }
+    } else if (directive === "disallow" && value && currentAgents.length > 0) {
+      for (const agent of currentAgents) {
+        agentGroups.get(agent).disallow.push(value);
+      }
+    } else if (directive === "allow" && value && currentAgents.length > 0) {
+      for (const agent of currentAgents) {
+        agentGroups.get(agent).allow.push(value);
+      }
+    } else if (directive !== "disallow" && directive !== "allow") {
+      currentAgents = [];
+    }
+  }
+  const specific = agentGroups.get(userAgent.toLowerCase());
+  if (specific && (specific.disallow.length > 0 || specific.allow.length > 0)) {
+    return specific;
+  }
+  return agentGroups.get("*") ?? { disallow: [], allow: [] };
+}
+function isBlockedByRobots(urlPath, rules) {
+  let longestDisallow = "";
+  for (const pattern of rules.disallow) {
+    if (urlPath.startsWith(pattern) && pattern.length > longestDisallow.length) {
+      longestDisallow = pattern;
+    }
+  }
+  if (!longestDisallow) return false;
+  let longestAllow = "";
+  for (const pattern of rules.allow) {
+    if (urlPath.startsWith(pattern) && pattern.length > longestAllow.length) {
+      longestAllow = pattern;
+    }
+  }
+  return longestAllow.length < longestDisallow.length;
+}
+async function loadRobotsTxtFromDir(dir) {
+  try {
+    const content = await fs9.readFile(path11.join(dir, "robots.txt"), "utf8");
+    return parseRobotsTxt(content);
+  } catch {
+    return null;
+  }
+}
+async function fetchRobotsTxt(baseUrl) {
+  try {
+    const url = new URL("/robots.txt", baseUrl).href;
+    const response = await fetch(url);
+    if (!response.ok) return null;
+    const content = await response.text();
+    return parseRobotsTxt(content);
+  } catch {
+    return null;
+  }
+}
+
+// src/search/ranking.ts
+function nonNegativeOrZero(value) {
+  if (!Number.isFinite(value)) {
+    return 0;
+  }
+  return Math.max(0, value);
+}
+function rankHits(hits, config) {
+  return hits.map((hit) => {
+    let score = Number.isFinite(hit.score) ? hit.score : Number.NEGATIVE_INFINITY;
+    if (config.ranking.enableIncomingLinkBoost) {
+      const incomingBoost = Math.log(1 + nonNegativeOrZero(hit.metadata.incomingLinks));
+      score += incomingBoost * config.ranking.weights.incomingLinks;
+    }
+    if (config.ranking.enableDepthBoost) {
+      const depthBoost = 1 / (1 + nonNegativeOrZero(hit.metadata.depth));
+      score += depthBoost * config.ranking.weights.depth;
+    }
+    return {
+      hit,
+      finalScore: Number.isFinite(score) ? score : Number.NEGATIVE_INFINITY
+    };
+  }).sort((a, b) => {
+    const delta = b.finalScore - a.finalScore;
+    return Number.isNaN(delta) ? 0 : delta;
+  });
+}
+function findPageWeight(url, pageWeights) {
+  let bestPattern = "";
+  let bestWeight = 1;
+  for (const [pattern, weight] of Object.entries(pageWeights)) {
+    if (matchUrlPattern(url, pattern) && pattern.length > bestPattern.length) {
+      bestPattern = pattern;
+      bestWeight = weight;
+    }
+  }
+  return bestWeight;
+}
+function aggregateByPage(ranked, config) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const hit of ranked) {
+    const url = hit.hit.metadata.url;
+    const group = groups.get(url);
+    if (group) group.push(hit);
+    else groups.set(url, [hit]);
+  }
+  const { aggregationCap, aggregationDecay } = config.ranking;
+  const pages = [];
+  for (const [url, chunks] of groups) {
+    chunks.sort((a, b) => {
+      const delta = b.finalScore - a.finalScore;
+      return Number.isNaN(delta) ? 0 : delta;
+    });
+    const best = chunks[0];
+    const maxScore = Number.isFinite(best.finalScore) ? best.finalScore : Number.NEGATIVE_INFINITY;
+    const topChunks = chunks.slice(0, aggregationCap);
+    let aggregationBonus = 0;
+    for (let i = 1; i < topChunks.length; i++) {
+      const chunkScore = Number.isFinite(topChunks[i].finalScore) ? topChunks[i].finalScore : 0;
+      aggregationBonus += chunkScore * Math.pow(aggregationDecay, i);
+    }
+    let pageScore = maxScore + aggregationBonus * config.ranking.weights.aggregation;
+    const pageWeight = findPageWeight(url, config.ranking.pageWeights);
+    if (pageWeight === 0) continue;
+    if (pageWeight !== 1) {
+      pageScore *= pageWeight;
+    }
+    pages.push({
+      url,
+      title: best.hit.metadata.title,
+      routeFile: best.hit.metadata.routeFile,
+      pageScore: Number.isFinite(pageScore) ? pageScore : Number.NEGATIVE_INFINITY,
+      bestChunk: best,
+      matchingChunks: chunks
+    });
+  }
+  return pages.sort((a, b) => {
+    const delta = b.pageScore - a.pageScore;
+    return Number.isNaN(delta) ? 0 : delta;
+  });
+}
+
 // src/utils/time.ts
 function nowIso() {
   return (/* @__PURE__ */ new Date()).toISOString();
@@ -2466,9 +2673,10 @@ function hrTimeMs(start) {
 
 // src/indexing/pipeline.ts
 var EMBEDDING_PRICE_PER_1K_TOKENS_USD = {
-  "jina-embeddings-v3": 2e-5
+  "jina-embeddings-v3": 2e-5,
+  "jina-embeddings-v5-text-small": 5e-5
 };
-var DEFAULT_EMBEDDING_PRICE_PER_1K = 2e-5;
+var DEFAULT_EMBEDDING_PRICE_PER_1K = 5e-5;
 var IndexPipeline = class _IndexPipeline {
   cwd;
   config;
@@ -2483,7 +2691,7 @@ var IndexPipeline = class _IndexPipeline {
     this.logger = options.logger;
   }
   static async create(options = {}) {
-    const cwd = path11.resolve(options.cwd ?? process.cwd());
+    const cwd = path12.resolve(options.cwd ?? process.cwd());
     const config = options.config ?? await loadConfig({ cwd, configPath: options.configPath });
     const embeddings = options.embeddingsProvider ?? createEmbeddingsProvider(config);
     const vectorStore = options.vectorStore ?? await createVectorStore(config, cwd);
@@ -2546,6 +2754,53 @@ var IndexPipeline = class _IndexPipeline {
     }
     stageEnd("source", sourceStart);
     this.logger.info(`Loaded ${sourcePages.length} page${sourcePages.length === 1 ? "" : "s"} (${stageTimingsMs["source"]}ms)`);
+    const filterStart = stageStart();
+    let filteredSourcePages = sourcePages;
+    if (this.config.exclude.length > 0) {
+      const beforeExclude = filteredSourcePages.length;
+      filteredSourcePages = filteredSourcePages.filter((p) => {
+        const url = normalizeUrlPath(p.url);
+        if (matchUrlPatterns(url, this.config.exclude)) {
+          this.logger.debug(`Excluding ${url} (matched exclude pattern)`);
+          return false;
+        }
+        return true;
+      });
+      const excludedCount = beforeExclude - filteredSourcePages.length;
+      if (excludedCount > 0) {
+        this.logger.info(`Excluded ${excludedCount} page${excludedCount === 1 ? "" : "s"} by config exclude patterns`);
+      }
+    }
+    if (this.config.respectRobotsTxt) {
+      let robotsRules = null;
+      if (sourceMode === "static-output") {
+        robotsRules = await loadRobotsTxtFromDir(
+          path12.resolve(this.cwd, this.config.source.staticOutputDir)
+        );
+      } else if (sourceMode === "build" && this.config.source.build) {
+        robotsRules = await loadRobotsTxtFromDir(
+          path12.resolve(this.cwd, this.config.source.build.outputDir)
+        );
+      } else if (sourceMode === "crawl" && this.config.source.crawl) {
+        robotsRules = await fetchRobotsTxt(this.config.source.crawl.baseUrl);
+      }
+      if (robotsRules) {
+        const beforeRobots = filteredSourcePages.length;
+        filteredSourcePages = filteredSourcePages.filter((p) => {
+          const url = normalizeUrlPath(p.url);
+          if (isBlockedByRobots(url, robotsRules)) {
+            this.logger.debug(`Excluding ${url} (blocked by robots.txt)`);
+            return false;
+          }
+          return true;
+        });
+        const robotsExcluded = beforeRobots - filteredSourcePages.length;
+        if (robotsExcluded > 0) {
+          this.logger.info(`Excluded ${robotsExcluded} page${robotsExcluded === 1 ? "" : "s"} by robots.txt`);
+        }
+      }
+    }
+    stageEnd("filter", filterStart);
     const routeStart = stageStart();
     const routePatterns = await buildRoutePatterns(this.cwd);
     stageEnd("route_map", routeStart);
@@ -2553,7 +2808,7 @@ var IndexPipeline = class _IndexPipeline {
     const extractStart = stageStart();
     this.logger.info("Extracting content...");
     const extractedPages = [];
-    for (const sourcePage of sourcePages) {
+    for (const sourcePage of filteredSourcePages) {
       const extracted = sourcePage.html ? extractFromHtml(sourcePage.url, sourcePage.html, this.config) : extractFromMarkdown(sourcePage.url, sourcePage.markdown ?? "", sourcePage.title);
       if (!extracted) {
         this.logger.warn(
@@ -2579,16 +2834,29 @@ var IndexPipeline = class _IndexPipeline {
       seenUrls.add(page.url);
       uniquePages.push(page);
     }
+    const indexablePages = [];
+    for (const page of uniquePages) {
+      const effectiveWeight = page.weight ?? findPageWeight(page.url, this.config.ranking.pageWeights);
+      if (effectiveWeight === 0) {
+        this.logger.debug(`Excluding ${page.url} (zero weight)`);
+        continue;
+      }
+      indexablePages.push(page);
+    }
+    const zeroWeightCount = uniquePages.length - indexablePages.length;
+    if (zeroWeightCount > 0) {
+      this.logger.info(`Excluded ${zeroWeightCount} page${zeroWeightCount === 1 ? "" : "s"} with zero weight`);
+    }
     stageEnd("extract", extractStart);
-    const skippedPages = sourcePages.length - uniquePages.length;
-    this.logger.info(`Extracted ${uniquePages.length} page${uniquePages.length === 1 ? "" : "s"}${skippedPages > 0 ? ` (${skippedPages} skipped)` : ""} (${stageTimingsMs["extract"]}ms)`);
+    const skippedPages = filteredSourcePages.length - indexablePages.length;
+    this.logger.info(`Extracted ${indexablePages.length} page${indexablePages.length === 1 ? "" : "s"}${skippedPages > 0 ? ` (${skippedPages} skipped)` : ""} (${stageTimingsMs["extract"]}ms)`);
     const linkStart = stageStart();
-    const pageSet = new Set(uniquePages.map((page) => normalizeUrlPath(page.url)));
+    const pageSet = new Set(indexablePages.map((page) => normalizeUrlPath(page.url)));
     const incomingLinkCount = /* @__PURE__ */ new Map();
-    for (const page of uniquePages) {
+    for (const page of indexablePages) {
       incomingLinkCount.set(page.url, incomingLinkCount.get(page.url) ?? 0);
     }
-    for (const page of uniquePages) {
+    for (const page of indexablePages) {
       for (const outgoing of page.outgoingLinks) {
         if (!pageSet.has(outgoing)) {
           continue;
@@ -2612,7 +2880,7 @@ var IndexPipeline = class _IndexPipeline {
         });
       }
     }
-    for (const page of uniquePages) {
+    for (const page of indexablePages) {
       const routeMatch = precomputedRoutes.get(normalizeUrlPath(page.url)) ?? mapUrlToRoute(page.url, routePatterns);
       if (routeMatch.routeResolution === "best-effort") {
         if (this.config.source.strictRouteMapping) {
@@ -2838,7 +3106,7 @@ import { createMcpExpressApp } from "@modelcontextprotocol/sdk/server/express.js
 import { z as z3 } from "zod";
 
 // src/search/engine.ts
-import path12 from "path";
+import path13 from "path";
 import { z as z2 } from "zod";
 
 // src/rerank/jina.ts
@@ -2854,7 +3122,7 @@ var JinaReranker = class {
   constructor(options) {
     this.apiKey = options.apiKey;
     this.model = options.model;
-    this.maxRetries = options.maxRetries ?? 4;
+    this.maxRetries = options.maxRetries ?? 2;
   }
   async rerank(query, candidates, topN) {
     if (candidates.length === 0) {
@@ -2864,7 +3132,8 @@ var JinaReranker = class {
       model: this.model,
       query,
       documents: candidates.map((candidate) => candidate.text),
-      top_n: topN ?? candidates.length
+      top_n: topN ?? candidates.length,
+      return_documents: false
     };
     let attempt = 0;
     while (attempt <= this.maxRetries) {
@@ -2937,98 +3206,6 @@ function createReranker(config) {
   });
 }
 
-// src/search/ranking.ts
-function nonNegativeOrZero(value) {
-  if (!Number.isFinite(value)) {
-    return 0;
-  }
-  return Math.max(0, value);
-}
-function rankHits(hits, config) {
-  return hits.map((hit) => {
-    let score = Number.isFinite(hit.score) ? hit.score : Number.NEGATIVE_INFINITY;
-    if (config.ranking.enableIncomingLinkBoost) {
-      const incomingBoost = Math.log(1 + nonNegativeOrZero(hit.metadata.incomingLinks));
-      score += incomingBoost * config.ranking.weights.incomingLinks;
-    }
-    if (config.ranking.enableDepthBoost) {
-      const depthBoost = 1 / (1 + nonNegativeOrZero(hit.metadata.depth));
-      score += depthBoost * config.ranking.weights.depth;
-    }
-    return {
-      hit,
-      finalScore: Number.isFinite(score) ? score : Number.NEGATIVE_INFINITY
-    };
-  }).sort((a, b) => {
-    const delta = b.finalScore - a.finalScore;
-    return Number.isNaN(delta) ? 0 : delta;
-  });
-}
-function findPageWeight(url, pageWeights) {
-  const norm = (p) => p !== "/" && p.endsWith("/") ? p.slice(0, -1) : p;
-  const normalizedUrl = norm(url);
-  for (const [pattern, weight] of Object.entries(pageWeights)) {
-    if (norm(pattern) === normalizedUrl) {
-      return weight;
-    }
-  }
-  let bestPrefix = "";
-  let bestWeight = 1;
-  for (const [pattern, weight] of Object.entries(pageWeights)) {
-    const normalizedPattern = norm(pattern);
-    if (normalizedPattern === "/") continue;
-    const prefix = `${normalizedPattern}/`;
-    if (normalizedUrl.startsWith(prefix) && prefix.length > bestPrefix.length) {
-      bestPrefix = prefix;
-      bestWeight = weight;
-    }
-  }
-  return bestWeight;
-}
-function aggregateByPage(ranked, config) {
-  const groups = /* @__PURE__ */ new Map();
-  for (const hit of ranked) {
-    const url = hit.hit.metadata.url;
-    const group = groups.get(url);
-    if (group) group.push(hit);
-    else groups.set(url, [hit]);
-  }
-  const { aggregationCap, aggregationDecay } = config.ranking;
-  const pages = [];
-  for (const [url, chunks] of groups) {
-    chunks.sort((a, b) => {
-      const delta = b.finalScore - a.finalScore;
-      return Number.isNaN(delta) ? 0 : delta;
-    });
-    const best = chunks[0];
-    const maxScore = Number.isFinite(best.finalScore) ? best.finalScore : Number.NEGATIVE_INFINITY;
-    const topChunks = chunks.slice(0, aggregationCap);
-    let aggregationBonus = 0;
-    for (let i = 1; i < topChunks.length; i++) {
-      const chunkScore = Number.isFinite(topChunks[i].finalScore) ? topChunks[i].finalScore : 0;
-      aggregationBonus += chunkScore * Math.pow(aggregationDecay, i);
-    }
-    let pageScore = maxScore + aggregationBonus * config.ranking.weights.aggregation;
-    const pageWeight = findPageWeight(url, config.ranking.pageWeights);
-    if (pageWeight === 0) continue;
-    if (pageWeight !== 1) {
-      pageScore *= pageWeight;
-    }
-    pages.push({
-      url,
-      title: best.hit.metadata.title,
-      routeFile: best.hit.metadata.routeFile,
-      pageScore: Number.isFinite(pageScore) ? pageScore : Number.NEGATIVE_INFINITY,
-      bestChunk: best,
-      matchingChunks: chunks
-    });
-  }
-  return pages.sort((a, b) => {
-    const delta = b.pageScore - a.pageScore;
-    return Number.isNaN(delta) ? 0 : delta;
-  });
-}
-
 // src/search/engine.ts
 var requestSchema = z2.object({
   q: z2.string().trim().min(1),
@@ -3037,7 +3214,8 @@ var requestSchema = z2.object({
   pathPrefix: z2.string().optional(),
   tags: z2.array(z2.string()).optional(),
   rerank: z2.boolean().optional(),
-  groupBy: z2.enum(["page", "chunk"]).optional()
+  groupBy: z2.enum(["page", "chunk"]).optional(),
+  stream: z2.boolean().optional()
 });
 var SearchEngine = class _SearchEngine {
   cwd;
@@ -3053,7 +3231,7 @@ var SearchEngine = class _SearchEngine {
     this.reranker = options.reranker;
   }
   static async create(options = {}) {
-    const cwd = path12.resolve(options.cwd ?? process.cwd());
+    const cwd = path13.resolve(options.cwd ?? process.cwd());
     const config = options.config ?? await loadConfig({ cwd, configPath: options.configPath });
     const embeddings = options.embeddingsProvider ?? createEmbeddingsProvider(config);
     const vectorStore = options.vectorStore ?? await createVectorStore(config, cwd);
@@ -3110,7 +3288,103 @@ var SearchEngine = class _SearchEngine {
       rerankMs = hrTimeMs(rerankStart);
       usedRerank = true;
     }
-    let results;
+    const results = this.buildResults(ordered, topK, groupByPage);
+    return {
+      q: input.q,
+      scope: resolvedScope.scopeName,
+      results,
+      meta: {
+        timingsMs: {
+          embed: Math.round(embedMs),
+          vector: Math.round(vectorMs),
+          rerank: Math.round(rerankMs),
+          total: Math.round(hrTimeMs(totalStart))
+        },
+        usedRerank,
+        modelId: this.config.embeddings.model
+      }
+    };
+  }
+  async *searchStreaming(request) {
+    const parsed = requestSchema.safeParse(request);
+    if (!parsed.success) {
+      throw new SearchSocketError("INVALID_REQUEST", parsed.error.issues[0]?.message ?? "Invalid request", 400);
+    }
+    const input = parsed.data;
+    const wantsRerank = Boolean(input.rerank);
+    if (!wantsRerank) {
+      const response = await this.search(request);
+      yield { phase: "initial", data: response };
+      return;
+    }
+    const totalStart = process.hrtime.bigint();
+    const resolvedScope = resolveScope(this.config, input.scope);
+    await this.assertModelCompatibility(resolvedScope);
+    const topK = input.topK ?? 10;
+    const groupByPage = (input.groupBy ?? "page") === "page";
+    const candidateK = groupByPage ? Math.max(topK * 10, 50) : Math.max(50, topK);
+    const embedStart = process.hrtime.bigint();
+    const queryEmbeddings = await this.embeddings.embedTexts([input.q], this.config.embeddings.model, "retrieval.query");
+    const queryVector = queryEmbeddings[0];
+    if (!queryVector || queryVector.length === 0 || queryVector.some((value) => !Number.isFinite(value))) {
+      throw new SearchSocketError("VECTOR_BACKEND_UNAVAILABLE", "Unable to create query embedding.");
+    }
+    const embedMs = hrTimeMs(embedStart);
+    const vectorStart = process.hrtime.bigint();
+    const hits = await this.vectorStore.query(
+      queryVector,
+      {
+        topK: candidateK,
+        pathPrefix: input.pathPrefix,
+        tags: input.tags
+      },
+      resolvedScope
+    );
+    const vectorMs = hrTimeMs(vectorStart);
+    const ranked = rankHits(hits, this.config);
+    const initialResults = this.buildResults(ranked, topK, groupByPage);
+    yield {
+      phase: "initial",
+      data: {
+        q: input.q,
+        scope: resolvedScope.scopeName,
+        results: initialResults,
+        meta: {
+          timingsMs: {
+            embed: Math.round(embedMs),
+            vector: Math.round(vectorMs),
+            rerank: 0,
+            total: Math.round(hrTimeMs(totalStart))
+          },
+          usedRerank: false,
+          modelId: this.config.embeddings.model
+        }
+      }
+    };
+    const rerankStart = process.hrtime.bigint();
+    const reranked = await this.rerankHits(input.q, ranked, topK);
+    const rerankMs = hrTimeMs(rerankStart);
+    const rerankedResults = this.buildResults(reranked, topK, groupByPage);
+    yield {
+      phase: "reranked",
+      data: {
+        q: input.q,
+        scope: resolvedScope.scopeName,
+        results: rerankedResults,
+        meta: {
+          timingsMs: {
+            embed: Math.round(embedMs),
+            vector: Math.round(vectorMs),
+            rerank: Math.round(rerankMs),
+            total: Math.round(hrTimeMs(totalStart))
+          },
+          usedRerank: true,
+          modelId: this.config.embeddings.model
+        }
+      }
+    };
+  }
+  buildResults(ordered, topK, groupByPage) {
     const minScore = this.config.ranking.minScore;
     if (groupByPage) {
       let pages = aggregateByPage(ordered, this.config);
@@ -3118,10 +3392,10 @@ var SearchEngine = class _SearchEngine {
         pages = pages.filter((p) => p.pageScore >= minScore);
       }
       const minRatio = this.config.ranking.minChunkScoreRatio;
-      results = pages.slice(0, topK).map((page) => {
+      return pages.slice(0, topK).map((page) => {
         const bestScore = page.bestChunk.finalScore;
-        const minScore2 = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
-        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minScore2).slice(0, 5);
+        const minChunkScore = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
+        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minChunkScore).slice(0, 5);
         return {
           url: page.url,
           title: page.title,
@@ -3138,10 +3412,11 @@ var SearchEngine = class _SearchEngine {
         };
       });
     } else {
+      let filtered = ordered;
       if (minScore > 0) {
-        ordered = ordered.filter((entry) => entry.finalScore >= minScore);
+        filtered = ordered.filter((entry) => entry.finalScore >= minScore);
       }
-      results = ordered.slice(0, topK).map(({ hit, finalScore }) => ({
+      return filtered.slice(0, topK).map(({ hit, finalScore }) => ({
         url: hit.metadata.url,
         title: hit.metadata.title,
         sectionTitle: hit.metadata.sectionTitle || void 0,
@@ -3150,21 +3425,6 @@ var SearchEngine = class _SearchEngine {
         routeFile: hit.metadata.routeFile
       }));
     }
-    return {
-      q: input.q,
-      scope: resolvedScope.scopeName,
-      results,
-      meta: {
-        timingsMs: {
-          embed: Math.round(embedMs),
-          vector: Math.round(vectorMs),
-          rerank: Math.round(rerankMs),
-          total: Math.round(hrTimeMs(totalStart))
-        },
-        usedRerank,
-        modelId: this.config.embeddings.model
-      }
-    };
   }
   async getPage(pathOrUrl, scope) {
     const resolvedScope = resolveScope(this.config, scope);
@@ -3236,6 +3496,7 @@ var SearchEngine = class _SearchEngine {
     const MAX_CHUNKS_PER_PAGE = 5;
     const MIN_CHUNKS_PER_PAGE = 1;
     const MIN_CHUNK_SCORE_RATIO = 0.5;
+    const MAX_DOC_CHARS = 2e3;
     const pageCandidates = [];
     for (const [url, chunks] of pageGroups) {
       const byScore = [...chunks].sort((a, b) => b.finalScore - a.finalScore);
@@ -3255,12 +3516,18 @@ var SearchEngine = class _SearchEngine {
       }
       const body = selected.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
       parts.push(body);
-      pageCandidates.push({ id: url, text: parts.join("\n\n") });
+      let text = parts.join("\n\n");
+      if (text.length > MAX_DOC_CHARS) {
+        text = text.slice(0, MAX_DOC_CHARS);
+      }
+      pageCandidates.push({ id: url, text });
     }
+    const maxCandidates = Math.max(topK, this.config.rerank.topN);
+    const cappedCandidates = pageCandidates.slice(0, maxCandidates);
     const reranked = await this.reranker.rerank(
       query,
-      pageCandidates,
-      Math.max(topK, this.config.rerank.topN)
+      cappedCandidates,
+      maxCandidates
     );
     const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {
@@ -3282,7 +3549,7 @@ var SearchEngine = class _SearchEngine {
 };
 
 // src/mcp/server.ts
-function createServer(engine) {
+function createServer(engine, config) {
   const server = new McpServer({
     name: "searchsocket-mcp",
     version: "0.1.0"
@@ -3290,14 +3557,15 @@ function createServer(engine) {
   server.registerTool(
     "search",
     {
-      description: "Semantic site search. Returns url/title/snippet/score/routeFile for each match. Supports optional scope, pathPrefix, tags, and topK.",
+      description: "Semantic site search. Returns url/title/snippet/score/routeFile for each match. Supports optional scope, pathPrefix, tags, topK, and rerank. Enable rerank for better relevance on natural-language queries.",
       inputSchema: {
         query: z3.string().min(1),
         scope: z3.string().optional(),
         topK: z3.number().int().positive().max(100).optional(),
         pathPrefix: z3.string().optional(),
         tags: z3.array(z3.string()).optional(),
-        groupBy: z3.enum(["page", "chunk"]).optional()
+        groupBy: z3.enum(["page", "chunk"]).optional(),
+        rerank: z3.boolean().optional().describe("Enable reranking for better relevance (uses Jina Reranker). Defaults to true when rerank is enabled in config.")
       }
     },
     async (input) => {
@@ -3307,7 +3575,8 @@ function createServer(engine) {
         scope: input.scope,
         pathPrefix: input.pathPrefix,
         tags: input.tags,
-        groupBy: input.groupBy
+        groupBy: input.groupBy,
+        rerank: input.rerank ?? config.rerank.enabled
       });
       return {
         content: [
@@ -3435,10 +3704,10 @@ async function runMcpServer(options = {}) {
     config
   });
   if (resolvedTransport === "http") {
-    await startHttpServer(() => createServer(engine), config, options);
+    await startHttpServer(() => createServer(engine, config), config, options);
     return;
   }
-  const server = createServer(engine);
+  const server = createServer(engine, config);
   const stdioTransport = new StdioServerTransport();
   await server.connect(stdioTransport);
 }
@@ -3507,7 +3776,7 @@ function collectWatchPaths(config, cwd) {
   const paths = ["src/routes/**"];
   if (config.source.mode === "content-files" && config.source.contentFiles) {
     for (const pattern of config.source.contentFiles.globs) {
-      paths.push(path13.join(config.source.contentFiles.baseDir, pattern));
+      paths.push(path14.join(config.source.contentFiles.baseDir, pattern));
     }
   }
   if (config.source.mode === "static-output") {
@@ -3520,15 +3789,15 @@ function collectWatchPaths(config, cwd) {
     paths.push("searchsocket.config.ts");
     paths.push(config.source.build.outputDir);
   }
-  return paths.map((value) => path13.resolve(cwd, value));
+  return paths.map((value) => path14.resolve(cwd, value));
 }
 function ensureStateDir(cwd) {
-  const target = path13.join(cwd, ".searchsocket");
-  fs9.mkdirSync(target, { recursive: true });
+  const target = path14.join(cwd, ".searchsocket");
+  fs10.mkdirSync(target, { recursive: true });
   return target;
 }
 function ensureGitignore(cwd) {
-  const gitignorePath = path13.join(cwd, ".gitignore");
+  const gitignorePath = path14.join(cwd, ".gitignore");
   const entries = [
     ".searchsocket/vectors.db",
     ".searchsocket/vectors.db-shm",
@@ -3537,8 +3806,8 @@ function ensureGitignore(cwd) {
     ".searchsocket/registry.json"
   ];
   let content = "";
-  if (fs9.existsSync(gitignorePath)) {
-    content = fs9.readFileSync(gitignorePath, "utf8");
+  if (fs10.existsSync(gitignorePath)) {
+    content = fs10.readFileSync(gitignorePath, "utf8");
   }
   const lines = content.split("\n");
   const missing = entries.filter((entry) => !lines.some((line) => line.trim() === entry));
@@ -3549,10 +3818,10 @@ function ensureGitignore(cwd) {
 # SearchSocket local state
 ${missing.join("\n")}
 `;
-  fs9.writeFileSync(gitignorePath, content.trimEnd() + block, "utf8");
+  fs10.writeFileSync(gitignorePath, content.trimEnd() + block, "utf8");
 }
 function readScopesFromFile(filePath) {
-  const raw = fs9.readFileSync(filePath, "utf8");
+  const raw = fs10.readFileSync(filePath, "utf8");
   return new Set(
     raw.split(/\r?\n/).map((line) => line.trim()).filter(Boolean)
   );
@@ -3576,8 +3845,8 @@ function readRemoteGitBranches(cwd) {
   }
 }
 async function loadResolvedConfigForDev(cwd, configPath) {
-  const resolvedConfigPath = path13.resolve(cwd, configPath ?? "searchsocket.config.ts");
-  if (fs9.existsSync(resolvedConfigPath)) {
+  const resolvedConfigPath = path14.resolve(cwd, configPath ?? "searchsocket.config.ts");
+  if (fs10.existsSync(resolvedConfigPath)) {
     return loadConfig({ cwd, configPath });
   }
   return mergeConfig(cwd, {});
@@ -3624,7 +3893,7 @@ var program = new Command();
 program.name("searchsocket").description("Semantic site search and MCP retrieval for SvelteKit").version(package_default.version).option("-C, --cwd <path>", "working directory", process.cwd()).option("--config <path>", "config path (defaults to searchsocket.config.ts)");
 program.command("init").description("Create searchsocket.config.ts and .searchsocket state directory").action(async (_opts, command) => {
   const root = getRootOptions(command).cwd ?? process.cwd();
-  const cwd = path13.resolve(root);
+  const cwd = path14.resolve(root);
   const configPath = writeMinimalConfig(cwd);
   const stateDir = ensureStateDir(cwd);
   ensureGitignore(cwd);
@@ -3644,13 +3913,13 @@ program.command("init").description("Create searchsocket.config.ts and .searchso
 });
 program.command("index").description("Index site content into markdown mirror + vector store").option("--scope <name>", "scope override").option("--changed-only", "only process changed chunks", true).option("--no-changed-only", "re-index regardless of previous manifest").option("--force", "force full mirror rebuild and re-upsert", false).option("--dry-run", "compute plan and cost, no API writes", false).option("--source <mode>", "source mode override: static-output|crawl|content-files|build").option("--max-pages <n>", "limit pages processed").option("--max-chunks <n>", "limit chunks processed").option("--quiet", "suppress all output except errors and warnings", false).option("--verbose", "verbose output", false).option("--json", "emit JSON logs and summary", false).action(async (opts, command) => {
   const rootOpts = getRootOptions(command);
-  const cwd = path13.resolve(rootOpts?.cwd ?? process.cwd());
+  const cwd = path14.resolve(rootOpts?.cwd ?? process.cwd());
   await runIndexCommand({
     cwd,
     configPath: rootOpts?.config,
     scope: opts.scope,
     changedOnly: opts.changedOnly,
-    force: opts.force,
+    force: opts.force || /^(1|true|yes)$/i.test(process.env.SEARCHSOCKET_FORCE_REINDEX ?? ""),
     dryRun: opts.dryRun,
     source: opts.source,
     maxPages: opts.maxPages ? parsePositiveInt(opts.maxPages, "--max-pages") : void 0,
@@ -3662,7 +3931,7 @@ program.command("index").description("Index site content into markdown mirror +
 });
 program.command("status").description("Show scope, indexing state, backend health, and recent cost estimate").option("--scope <name>", "scope override").action(async (opts, command) => {
   const rootOpts = getRootOptions(command);
-  const cwd = path13.resolve(rootOpts?.cwd ?? process.cwd());
+  const cwd = path14.resolve(rootOpts?.cwd ?? process.cwd());
   const config = await loadConfig({ cwd, configPath: rootOpts?.config });
   const scope = resolveScope(config, opts.scope);
   let vectorStore;
@@ -3740,7 +4009,7 @@ program.command("status").description("Show scope, indexing state, backend healt
 });
 program.command("dev").description("Watch content files/routes and incrementally reindex on changes").option("--scope <name>", "scope override").option("--mcp", "start MCP server (http transport) alongside watcher", false).option("--mcp-port <n>", "MCP HTTP port", "3338").option("--mcp-path <path>", "MCP HTTP path", "/mcp").option("--verbose", "verbose logs", false).action(async (opts, command) => {
   const rootOpts = getRootOptions(command);
-  const cwd = path13.resolve(rootOpts?.cwd ?? process.cwd());
+  const cwd = path14.resolve(rootOpts?.cwd ?? process.cwd());
   const config = await loadResolvedConfigForDev(cwd, rootOpts?.config);
   const watchPaths = collectWatchPaths(config, cwd);
   process.stdout.write("starting searchsocket dev watcher...\n");
@@ -3809,10 +4078,10 @@ ${watchPaths.map((entry) => `  - ${entry}`).join("\n")}
 });
 program.command("clean").description("Delete local state and optionally delete remote vectors for a scope").option("--scope <name>", "scope override").option("--remote", "delete remote scope vectors", false).action(async (opts, command) => {
   const rootOpts = getRootOptions(command);
-  const cwd = path13.resolve(rootOpts?.cwd ?? process.cwd());
+  const cwd = path14.resolve(rootOpts?.cwd ?? process.cwd());
   const config = await loadConfig({ cwd, configPath: rootOpts?.config });
   const scope = resolveScope(config, opts.scope);
-  const statePath = path13.join(cwd, config.state.dir);
+  const statePath = path14.join(cwd, config.state.dir);
   await fsp.rm(statePath, { recursive: true, force: true });
   process.stdout.write(`deleted local state directory: ${statePath}
 `);
@@ -3825,7 +4094,7 @@ program.command("clean").description("Delete local state and optionally delete r
 });
 program.command("prune").description("List/delete stale scopes (dry-run by default)").option("--apply", "apply deletions", false).option("--scopes-file <path>", "file containing active scopes").option("--older-than <duration>", "ttl cutoff like 30d").action(async (opts, command) => {
   const rootOpts = getRootOptions(command);
-  const cwd = path13.resolve(rootOpts?.cwd ?? process.cwd());
+  const cwd = path14.resolve(rootOpts?.cwd ?? process.cwd());
   const config = await loadConfig({ cwd, configPath: rootOpts?.config });
   const baseScope = resolveScope(config);
   let vectorStore;
@@ -3845,7 +4114,7 @@ program.command("prune").description("List/delete stale scopes (dry-run by defau
 `);
   let keepScopes = /* @__PURE__ */ new Set();
   if (opts.scopesFile) {
-    keepScopes = readScopesFromFile(path13.resolve(cwd, opts.scopesFile));
+    keepScopes = readScopesFromFile(path14.resolve(cwd, opts.scopesFile));
   } else {
     keepScopes = readRemoteGitBranches(cwd);
   }
@@ -3916,7 +4185,7 @@ program.command("prune").description("List/delete stale scopes (dry-run by defau
 });
 program.command("doctor").description("Validate config, env vars, provider connectivity, and local write access").action(async (_opts, command) => {
   const rootOpts = getRootOptions(command);
-  const cwd = path13.resolve(rootOpts?.cwd ?? process.cwd());
+  const cwd = path14.resolve(rootOpts?.cwd ?? process.cwd());
   const checks = [];
   let config = null;
   try {
@@ -3945,8 +4214,8 @@ program.command("doctor").description("Validate config, env vars, provider conne
       });
     }
     if (config.source.mode === "static-output") {
-      const outputDir = path13.resolve(cwd, config.source.staticOutputDir);
-      const exists = fs9.existsSync(outputDir);
+      const outputDir = path14.resolve(cwd, config.source.staticOutputDir);
+      const exists = fs10.existsSync(outputDir);
       checks.push({
         name: "source: static output dir",
         ok: exists,
@@ -3955,15 +4224,15 @@ program.command("doctor").description("Validate config, env vars, provider conne
     } else if (config.source.mode === "build") {
       const buildConfig = config.source.build;
       if (buildConfig) {
-        const manifestPath = path13.resolve(cwd, buildConfig.outputDir, "server", "manifest-full.js");
-        const manifestExists = fs9.existsSync(manifestPath);
+        const manifestPath = path14.resolve(cwd, buildConfig.outputDir, "server", "manifest-full.js");
+        const manifestExists = fs10.existsSync(manifestPath);
         checks.push({
           name: "source: build manifest",
           ok: manifestExists,
           details: manifestExists ? manifestPath : `${manifestPath} not found (run \`vite build\` first)`
         });
-        const viteBin = path13.resolve(cwd, "node_modules", ".bin", "vite");
-        const viteExists = fs9.existsSync(viteBin);
+        const viteBin = path14.resolve(cwd, "node_modules", ".bin", "vite");
+        const viteExists = fs10.existsSync(viteBin);
         checks.push({
           name: "source: vite binary",
           ok: viteExists,
@@ -3980,7 +4249,7 @@ program.command("doctor").description("Validate config, env vars, provider conne
       const contentConfig = config.source.contentFiles;
       if (contentConfig) {
         const fg4 = await import("fast-glob");
-        const baseDir = path13.resolve(cwd, contentConfig.baseDir);
+        const baseDir = path14.resolve(cwd, contentConfig.baseDir);
         const files = await fg4.default(contentConfig.globs, { cwd: baseDir, onlyFiles: true });
         checks.push({
           name: "source: content files",
@@ -4049,7 +4318,7 @@ program.command("doctor").description("Validate config, env vars, provider conne
     try {
       const scope = resolveScope(config);
       const { statePath } = ensureStateDirs(cwd, config.state.dir, scope);
-      const testPath = path13.join(statePath, ".write-test");
+      const testPath = path14.join(statePath, ".write-test");
       await fsp.writeFile(testPath, "ok\n", "utf8");
       await fsp.rm(testPath, { force: true });
       checks.push({ name: "state directory writable", ok: true });
@@ -4078,7 +4347,7 @@ program.command("doctor").description("Validate config, env vars, provider conne
 });
 program.command("mcp").description("Run SearchSocket MCP server").option("--transport <transport>", "stdio|http", "stdio").option("--port <n>", "HTTP port", "3338").option("--path <path>", "HTTP path", "/mcp").action(async (opts, command) => {
   const rootOpts = getRootOptions(command);
-  const cwd = path13.resolve(rootOpts?.cwd ?? process.cwd());
+  const cwd = path14.resolve(rootOpts?.cwd ?? process.cwd());
   await runMcpServer({
     cwd,
     configPath: rootOpts?.config,
@@ -4089,7 +4358,7 @@ program.command("mcp").description("Run SearchSocket MCP server").option("--tran
 });
 program.command("search").description("Quick local CLI search against indexed vectors").requiredOption("--q <query>", "search query").option("--scope <name>", "scope override").option("--top-k <n>", "top K results", "10").option("--path-prefix <prefix>", "path prefix filter").option("--rerank", "enable configured reranker", false).action(async (opts, command) => {
   const rootOpts = getRootOptions(command);
-  const cwd = path13.resolve(rootOpts?.cwd ?? process.cwd());
+  const cwd = path14.resolve(rootOpts?.cwd ?? process.cwd());
   const engine = await SearchEngine.create({
     cwd,
     configPath: rootOpts?.config
@@ -4105,7 +4374,7 @@ program.command("search").description("Quick local CLI search against indexed ve
 `);
 });
 async function main() {
-  dotenvConfig({ path: path13.resolve(process.cwd(), ".env") });
+  dotenvConfig({ path: path14.resolve(process.cwd(), ".env") });
   await program.parseAsync(process.argv);
 }
 main().catch((error) => {