searchsocket 0.3.2 → 0.4.0

package/dist/sveltekit.d.ts

@@ -1,4 +1,4 @@
-import { R as ResolvedSearchSocketConfig, b as SearchSocketConfig } from './types-BrG6XTUU.js';
+import { R as ResolvedSearchSocketConfig, d as SearchSocketConfig } from './types-z2dw3H6E.js';
 
 interface SearchSocketHandleOptions {
     configPath?: string;

package/dist/sveltekit.js

@@ -5009,32 +5009,32 @@ var require_URL = __commonJS({
           else
             return basepath.substring(0, lastslash + 1) + refpath;
         }
-        function remove_dot_segments(path14) {
-          if (!path14) return path14;
+        function remove_dot_segments(path15) {
+          if (!path15) return path15;
           var output = "";
-          while (path14.length > 0) {
-            if (path14 === "." || path14 === "..") {
-              path14 = "";
+          while (path15.length > 0) {
+            if (path15 === "." || path15 === "..") {
+              path15 = "";
               break;
             }
-            var twochars = path14.substring(0, 2);
-            var threechars = path14.substring(0, 3);
-            var fourchars = path14.substring(0, 4);
+            var twochars = path15.substring(0, 2);
+            var threechars = path15.substring(0, 3);
+            var fourchars = path15.substring(0, 4);
             if (threechars === "../") {
-              path14 = path14.substring(3);
+              path15 = path15.substring(3);
             } else if (twochars === "./") {
-              path14 = path14.substring(2);
+              path15 = path15.substring(2);
             } else if (threechars === "/./") {
-              path14 = "/" + path14.substring(3);
-            } else if (twochars === "/." && path14.length === 2) {
-              path14 = "/";
-            } else if (fourchars === "/../" || threechars === "/.." && path14.length === 3) {
-              path14 = "/" + path14.substring(4);
+              path15 = "/" + path15.substring(3);
+            } else if (twochars === "/." && path15.length === 2) {
+              path15 = "/";
+            } else if (fourchars === "/../" || threechars === "/.." && path15.length === 3) {
+              path15 = "/" + path15.substring(4);
               output = output.replace(/\/?[^\/]*$/, "");
             } else {
-              var segment = path14.match(/(\/?([^\/]*))/)[0];
+              var segment = path15.match(/(\/?([^\/]*))/)[0];
               output += segment;
-              path14 = path14.substring(segment.length);
+              path15 = path15.substring(segment.length);
             }
           }
           return output;
@@ -16598,6 +16598,8 @@ var searchSocketConfigSchema = z.object({
     envVar: z.string().min(1).optional(),
     sanitize: z.boolean().optional()
   }).optional(),
+  exclude: z.array(z.string()).optional(),
+  respectRobotsTxt: z.boolean().optional(),
   source: z.object({
     mode: z.enum(["static-output", "crawl", "content-files", "build"]).optional(),
     staticOutputDir: z.string().min(1).optional(),
@@ -16728,6 +16730,8 @@ function createDefaultConfig(projectId) {
       envVar: "SEARCHSOCKET_SCOPE",
       sanitize: true
     },
+    exclude: [],
+    respectRobotsTxt: true,
     source: {
       mode: "static-output",
       staticOutputDir: "build",
@@ -16758,7 +16762,7 @@ function createDefaultConfig(projectId) {
     },
     embeddings: {
       provider: "jina",
-      model: "jina-embeddings-v3",
+      model: "jina-embeddings-v5-text-small",
       apiKeyEnv: "JINA_API_KEY",
       batchSize: 64,
       concurrency: 4
@@ -16771,9 +16775,9 @@ function createDefaultConfig(projectId) {
       }
     },
     rerank: {
-      enabled: false,
+      enabled: true,
       topN: 20,
-      model: "jina-reranker-v2-base-multilingual"
+      model: "jina-reranker-v3"
     },
     ranking: {
       enableIncomingLinkBoost: true,
@@ -16892,6 +16896,8 @@ ${issues}`
       ...defaults.scope,
       ...parsed.scope
     },
+    exclude: parsed.exclude ?? defaults.exclude,
+    respectRobotsTxt: parsed.respectRobotsTxt ?? defaults.respectRobotsTxt,
     source: {
       ...defaults.source,
       ...parsed.source,
@@ -17242,7 +17248,7 @@ var JinaReranker = class {
   constructor(options) {
     this.apiKey = options.apiKey;
     this.model = options.model;
-    this.maxRetries = options.maxRetries ?? 4;
+    this.maxRetries = options.maxRetries ?? 2;
   }
   async rerank(query, candidates, topN) {
     if (candidates.length === 0) {
@@ -17252,7 +17258,8 @@ var JinaReranker = class {
       model: this.model,
       query,
       documents: candidates.map((candidate) => candidate.text),
-      top_n: topN ?? candidates.length
+      top_n: topN ?? candidates.length,
+      return_documents: false
     };
     let attempt = 0;
     while (attempt <= this.maxRetries) {
@@ -17855,6 +17862,36 @@ async function createVectorStore(config, cwd) {
   });
 }
 
+// src/utils/pattern.ts
+function matchUrlPattern(url, pattern) {
+  const norm = (p) => p !== "/" && p.endsWith("/") ? p.slice(0, -1) : p;
+  const normalizedUrl = norm(url);
+  const normalizedPattern = norm(pattern);
+  if (normalizedPattern.endsWith("/**")) {
+    const prefix = normalizedPattern.slice(0, -3);
+    if (prefix === "") {
+      return true;
+    }
+    return normalizedUrl === prefix || normalizedUrl.startsWith(prefix + "/");
+  }
+  if (normalizedPattern.endsWith("/*")) {
+    const prefix = normalizedPattern.slice(0, -2);
+    if (prefix === "") {
+      return normalizedUrl !== "/" && !normalizedUrl.slice(1).includes("/");
+    }
+    if (!normalizedUrl.startsWith(prefix + "/")) return false;
+    const rest = normalizedUrl.slice(prefix.length + 1);
+    return rest.length > 0 && !rest.includes("/");
+  }
+  return normalizedUrl === normalizedPattern;
+}
+function matchUrlPatterns(url, patterns) {
+  for (const pattern of patterns) {
+    if (matchUrlPattern(url, pattern)) return true;
+  }
+  return false;
+}
+
 // src/search/ranking.ts
 function nonNegativeOrZero(value) {
   if (!Number.isFinite(value)) {
@@ -17883,21 +17920,11 @@ function rankHits(hits, config) {
   });
 }
 function findPageWeight(url, pageWeights) {
-  const norm = (p) => p !== "/" && p.endsWith("/") ? p.slice(0, -1) : p;
-  const normalizedUrl = norm(url);
-  for (const [pattern, weight] of Object.entries(pageWeights)) {
-    if (norm(pattern) === normalizedUrl) {
-      return weight;
-    }
-  }
-  let bestPrefix = "";
+  let bestPattern = "";
   let bestWeight = 1;
   for (const [pattern, weight] of Object.entries(pageWeights)) {
-    const normalizedPattern = norm(pattern);
-    if (normalizedPattern === "/") continue;
-    const prefix = `${normalizedPattern}/`;
-    if (normalizedUrl.startsWith(prefix) && prefix.length > bestPrefix.length) {
-      bestPrefix = prefix;
+    if (matchUrlPattern(url, pattern) && pattern.length > bestPattern.length) {
+      bestPattern = pattern;
       bestWeight = weight;
     }
   }
@@ -17955,7 +17982,8 @@ var requestSchema = z.object({
   pathPrefix: z.string().optional(),
   tags: z.array(z.string()).optional(),
   rerank: z.boolean().optional(),
-  groupBy: z.enum(["page", "chunk"]).optional()
+  groupBy: z.enum(["page", "chunk"]).optional(),
+  stream: z.boolean().optional()
 });
 var SearchEngine = class _SearchEngine {
   cwd;
@@ -18028,7 +18056,103 @@ var SearchEngine = class _SearchEngine {
       rerankMs = hrTimeMs(rerankStart);
       usedRerank = true;
     }
-    let results;
+    const results = this.buildResults(ordered, topK, groupByPage);
+    return {
+      q: input.q,
+      scope: resolvedScope.scopeName,
+      results,
+      meta: {
+        timingsMs: {
+          embed: Math.round(embedMs),
+          vector: Math.round(vectorMs),
+          rerank: Math.round(rerankMs),
+          total: Math.round(hrTimeMs(totalStart))
+        },
+        usedRerank,
+        modelId: this.config.embeddings.model
+      }
+    };
+  }
+  async *searchStreaming(request) {
+    const parsed = requestSchema.safeParse(request);
+    if (!parsed.success) {
+      throw new SearchSocketError("INVALID_REQUEST", parsed.error.issues[0]?.message ?? "Invalid request", 400);
+    }
+    const input = parsed.data;
+    const wantsRerank = Boolean(input.rerank);
+    if (!wantsRerank) {
+      const response = await this.search(request);
+      yield { phase: "initial", data: response };
+      return;
+    }
+    const totalStart = process.hrtime.bigint();
+    const resolvedScope = resolveScope(this.config, input.scope);
+    await this.assertModelCompatibility(resolvedScope);
+    const topK = input.topK ?? 10;
+    const groupByPage = (input.groupBy ?? "page") === "page";
+    const candidateK = groupByPage ? Math.max(topK * 10, 50) : Math.max(50, topK);
+    const embedStart = process.hrtime.bigint();
+    const queryEmbeddings = await this.embeddings.embedTexts([input.q], this.config.embeddings.model, "retrieval.query");
+    const queryVector = queryEmbeddings[0];
+    if (!queryVector || queryVector.length === 0 || queryVector.some((value) => !Number.isFinite(value))) {
+      throw new SearchSocketError("VECTOR_BACKEND_UNAVAILABLE", "Unable to create query embedding.");
+    }
+    const embedMs = hrTimeMs(embedStart);
+    const vectorStart = process.hrtime.bigint();
+    const hits = await this.vectorStore.query(
+      queryVector,
+      {
+        topK: candidateK,
+        pathPrefix: input.pathPrefix,
+        tags: input.tags
+      },
+      resolvedScope
+    );
+    const vectorMs = hrTimeMs(vectorStart);
+    const ranked = rankHits(hits, this.config);
+    const initialResults = this.buildResults(ranked, topK, groupByPage);
+    yield {
+      phase: "initial",
+      data: {
+        q: input.q,
+        scope: resolvedScope.scopeName,
+        results: initialResults,
+        meta: {
+          timingsMs: {
+            embed: Math.round(embedMs),
+            vector: Math.round(vectorMs),
+            rerank: 0,
+            total: Math.round(hrTimeMs(totalStart))
+          },
+          usedRerank: false,
+          modelId: this.config.embeddings.model
+        }
+      }
+    };
+    const rerankStart = process.hrtime.bigint();
+    const reranked = await this.rerankHits(input.q, ranked, topK);
+    const rerankMs = hrTimeMs(rerankStart);
+    const rerankedResults = this.buildResults(reranked, topK, groupByPage);
+    yield {
+      phase: "reranked",
+      data: {
+        q: input.q,
+        scope: resolvedScope.scopeName,
+        results: rerankedResults,
+        meta: {
+          timingsMs: {
+            embed: Math.round(embedMs),
+            vector: Math.round(vectorMs),
+            rerank: Math.round(rerankMs),
+            total: Math.round(hrTimeMs(totalStart))
+          },
+          usedRerank: true,
+          modelId: this.config.embeddings.model
+        }
+      }
+    };
+  }
+  buildResults(ordered, topK, groupByPage) {
     const minScore = this.config.ranking.minScore;
     if (groupByPage) {
       let pages = aggregateByPage(ordered, this.config);
@@ -18036,10 +18160,10 @@ var SearchEngine = class _SearchEngine {
         pages = pages.filter((p) => p.pageScore >= minScore);
       }
       const minRatio = this.config.ranking.minChunkScoreRatio;
-      results = pages.slice(0, topK).map((page) => {
+      return pages.slice(0, topK).map((page) => {
         const bestScore = page.bestChunk.finalScore;
-        const minScore2 = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
-        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minScore2).slice(0, 5);
+        const minChunkScore = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
+        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minChunkScore).slice(0, 5);
         return {
           url: page.url,
           title: page.title,
@@ -18056,10 +18180,11 @@ var SearchEngine = class _SearchEngine {
         };
       });
     } else {
+      let filtered = ordered;
       if (minScore > 0) {
-        ordered = ordered.filter((entry) => entry.finalScore >= minScore);
+        filtered = ordered.filter((entry) => entry.finalScore >= minScore);
       }
-      results = ordered.slice(0, topK).map(({ hit, finalScore }) => ({
+      return filtered.slice(0, topK).map(({ hit, finalScore }) => ({
         url: hit.metadata.url,
         title: hit.metadata.title,
         sectionTitle: hit.metadata.sectionTitle || void 0,
@@ -18068,21 +18193,6 @@ var SearchEngine = class _SearchEngine {
         routeFile: hit.metadata.routeFile
       }));
     }
-    return {
-      q: input.q,
-      scope: resolvedScope.scopeName,
-      results,
-      meta: {
-        timingsMs: {
-          embed: Math.round(embedMs),
-          vector: Math.round(vectorMs),
-          rerank: Math.round(rerankMs),
-          total: Math.round(hrTimeMs(totalStart))
-        },
-        usedRerank,
-        modelId: this.config.embeddings.model
-      }
-    };
   }
   async getPage(pathOrUrl, scope) {
     const resolvedScope = resolveScope(this.config, scope);
@@ -18154,6 +18264,7 @@ var SearchEngine = class _SearchEngine {
     const MAX_CHUNKS_PER_PAGE = 5;
     const MIN_CHUNKS_PER_PAGE = 1;
     const MIN_CHUNK_SCORE_RATIO = 0.5;
+    const MAX_DOC_CHARS = 2e3;
     const pageCandidates = [];
     for (const [url, chunks] of pageGroups) {
       const byScore = [...chunks].sort((a, b) => b.finalScore - a.finalScore);
@@ -18173,12 +18284,18 @@ var SearchEngine = class _SearchEngine {
       }
       const body = selected.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
       parts.push(body);
-      pageCandidates.push({ id: url, text: parts.join("\n\n") });
+      let text = parts.join("\n\n");
+      if (text.length > MAX_DOC_CHARS) {
+        text = text.slice(0, MAX_DOC_CHARS);
+      }
+      pageCandidates.push({ id: url, text });
     }
+    const maxCandidates = Math.max(topK, this.config.rerank.topN);
+    const cappedCandidates = pageCandidates.slice(0, maxCandidates);
     const reranked = await this.reranker.rerank(
       query,
-      pageCandidates,
-      Math.max(topK, this.config.rerank.topN)
+      cappedCandidates,
+      maxCandidates
     );
     const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {
@@ -18350,7 +18467,44 @@ function searchsocketHandle(options = {}) {
         throw new SearchSocketError("INVALID_REQUEST", "Malformed JSON request body", 400);
       }
       const engine = await getEngine();
-      const result = await engine.search(body);
+      const searchRequest = body;
+      if (searchRequest.stream && searchRequest.rerank) {
+        const encoder = new TextEncoder();
+        const stream = new ReadableStream({
+          async start(controller) {
+            try {
+              for await (const event2 of engine.searchStreaming(searchRequest)) {
+                const line = JSON.stringify(event2) + "\n";
+                controller.enqueue(encoder.encode(line));
+              }
+            } catch (streamError) {
+              const errorEvent = {
+                phase: "error",
+                data: {
+                  error: {
+                    code: streamError instanceof SearchSocketError ? streamError.code : "INTERNAL_ERROR",
+                    message: streamError instanceof Error ? streamError.message : "Unknown error"
+                  }
+                }
+              };
+              controller.enqueue(encoder.encode(JSON.stringify(errorEvent) + "\n"));
+            } finally {
+              controller.close();
+            }
+          }
+        });
+        return withCors(
+          new Response(stream, {
+            status: 200,
+            headers: {
+              "content-type": "application/x-ndjson"
+            }
+          }),
+          event.request,
+          config
+        );
+      }
+      const result = await engine.search(searchRequest);
       return withCors(
         new Response(JSON.stringify(result), {
           status: 200,
@@ -19579,6 +19733,17 @@ function extractFromHtml(url, html, config) {
   if ($(`[${config.extract.noindexAttr}]`).length > 0) {
     return null;
   }
+  const weightRaw = $("meta[name='searchsocket-weight']").attr("content")?.trim();
+  let weight;
+  if (weightRaw !== void 0) {
+    const parsed = Number(weightRaw);
+    if (Number.isFinite(parsed) && parsed >= 0) {
+      weight = parsed;
+    }
+  }
+  if (weight === 0) {
+    return null;
+  }
   const description = $("meta[name='description']").attr("content")?.trim() || $("meta[property='og:description']").attr("content")?.trim() || void 0;
   const keywordsRaw = $("meta[name='keywords']").attr("content")?.trim();
   const keywords = keywordsRaw ? keywordsRaw.split(",").map((k) => k.trim()).filter(Boolean) : void 0;
@@ -19634,7 +19799,8 @@ function extractFromHtml(url, html, config) {
     noindex: false,
     tags,
     description,
-    keywords
+    keywords,
+    weight
   };
 }
 function extractFromMarkdown(url, markdown, title) {
@@ -19647,6 +19813,14 @@ function extractFromMarkdown(url, markdown, title) {
   if (frontmatter.noindex === true || searchsocketMeta?.noindex === true) {
     return null;
   }
+  let mdWeight;
+  const rawWeight = searchsocketMeta?.weight ?? frontmatter.searchsocketWeight;
+  if (typeof rawWeight === "number" && Number.isFinite(rawWeight) && rawWeight >= 0) {
+    mdWeight = rawWeight;
+  }
+  if (mdWeight === 0) {
+    return null;
+  }
   const content = parsed.content;
   const normalized = normalizeMarkdown(content);
   if (!normalizeText(normalized)) {
@@ -19669,7 +19843,8 @@ function extractFromMarkdown(url, markdown, title) {
     noindex: false,
     tags: normalizeUrlPath(url).split("/").filter(Boolean).slice(0, 1),
     description: fmDescription,
-    keywords: fmKeywords
+    keywords: fmKeywords,
+    weight: mdWeight
   };
 }
 function yamlString(value) {
@@ -19938,15 +20113,7 @@ function expandDynamicUrl(url, value) {
   return url.replace(/\[\[?\.\.\.[^\]]+\]?\]|\[\[[^\]]+\]\]|\[[^\]]+\]/g, value);
 }
 function isExcluded(url, patterns) {
-  for (const pattern of patterns) {
-    if (pattern.endsWith("/*")) {
-      const prefix = pattern.slice(0, -1);
-      if (url.startsWith(prefix) || url === prefix.slice(0, -1)) return true;
-    } else if (url === pattern) {
-      return true;
-    }
-  }
-  return false;
+  return matchUrlPatterns(url, patterns);
 }
 function findFreePort() {
   return new Promise((resolve, reject) => {
@@ -20362,12 +20529,83 @@ async function loadStaticOutputPages(cwd, config, maxPages) {
   }
   return pages;
 }
+function parseRobotsTxt(content, userAgent = "Searchsocket") {
+  const lines = content.split(/\r?\n/);
+  const agentGroups = /* @__PURE__ */ new Map();
+  let currentAgents = [];
+  for (const rawLine of lines) {
+    const line = rawLine.replace(/#.*$/, "").trim();
+    if (!line) continue;
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) continue;
+    const directive = line.slice(0, colonIdx).trim().toLowerCase();
+    const value = line.slice(colonIdx + 1).trim();
+    if (directive === "user-agent") {
+      const agentName = value.toLowerCase();
+      currentAgents.push(agentName);
+      if (!agentGroups.has(agentName)) {
+        agentGroups.set(agentName, { disallow: [], allow: [] });
+      }
+    } else if (directive === "disallow" && value && currentAgents.length > 0) {
+      for (const agent of currentAgents) {
+        agentGroups.get(agent).disallow.push(value);
+      }
+    } else if (directive === "allow" && value && currentAgents.length > 0) {
+      for (const agent of currentAgents) {
+        agentGroups.get(agent).allow.push(value);
+      }
+    } else if (directive !== "disallow" && directive !== "allow") {
+      currentAgents = [];
+    }
+  }
+  const specific = agentGroups.get(userAgent.toLowerCase());
+  if (specific && (specific.disallow.length > 0 || specific.allow.length > 0)) {
+    return specific;
+  }
+  return agentGroups.get("*") ?? { disallow: [], allow: [] };
+}
+function isBlockedByRobots(urlPath, rules3) {
+  let longestDisallow = "";
+  for (const pattern of rules3.disallow) {
+    if (urlPath.startsWith(pattern) && pattern.length > longestDisallow.length) {
+      longestDisallow = pattern;
+    }
+  }
+  if (!longestDisallow) return false;
+  let longestAllow = "";
+  for (const pattern of rules3.allow) {
+    if (urlPath.startsWith(pattern) && pattern.length > longestAllow.length) {
+      longestAllow = pattern;
+    }
+  }
+  return longestAllow.length < longestDisallow.length;
+}
+async function loadRobotsTxtFromDir(dir) {
+  try {
+    const content = await fs4.readFile(path.join(dir, "robots.txt"), "utf8");
+    return parseRobotsTxt(content);
+  } catch {
+    return null;
+  }
+}
+async function fetchRobotsTxt(baseUrl) {
+  try {
+    const url = new URL("/robots.txt", baseUrl).href;
+    const response = await fetch(url);
+    if (!response.ok) return null;
+    const content = await response.text();
+    return parseRobotsTxt(content);
+  } catch {
+    return null;
+  }
+}
 
 // src/indexing/pipeline.ts
 var EMBEDDING_PRICE_PER_1K_TOKENS_USD = {
-  "jina-embeddings-v3": 2e-5
+  "jina-embeddings-v3": 2e-5,
+  "jina-embeddings-v5-text-small": 5e-5
 };
-var DEFAULT_EMBEDDING_PRICE_PER_1K = 2e-5;
+var DEFAULT_EMBEDDING_PRICE_PER_1K = 5e-5;
 var IndexPipeline = class _IndexPipeline {
   cwd;
   config;
@@ -20445,6 +20683,53 @@ var IndexPipeline = class _IndexPipeline {
     }
     stageEnd("source", sourceStart);
     this.logger.info(`Loaded ${sourcePages.length} page${sourcePages.length === 1 ? "" : "s"} (${stageTimingsMs["source"]}ms)`);
+    const filterStart = stageStart();
+    let filteredSourcePages = sourcePages;
+    if (this.config.exclude.length > 0) {
+      const beforeExclude = filteredSourcePages.length;
+      filteredSourcePages = filteredSourcePages.filter((p) => {
+        const url = normalizeUrlPath(p.url);
+        if (matchUrlPatterns(url, this.config.exclude)) {
+          this.logger.debug(`Excluding ${url} (matched exclude pattern)`);
+          return false;
+        }
+        return true;
+      });
+      const excludedCount = beforeExclude - filteredSourcePages.length;
+      if (excludedCount > 0) {
+        this.logger.info(`Excluded ${excludedCount} page${excludedCount === 1 ? "" : "s"} by config exclude patterns`);
+      }
+    }
+    if (this.config.respectRobotsTxt) {
+      let robotsRules = null;
+      if (sourceMode === "static-output") {
+        robotsRules = await loadRobotsTxtFromDir(
+          path.resolve(this.cwd, this.config.source.staticOutputDir)
+        );
+      } else if (sourceMode === "build" && this.config.source.build) {
+        robotsRules = await loadRobotsTxtFromDir(
+          path.resolve(this.cwd, this.config.source.build.outputDir)
+        );
+      } else if (sourceMode === "crawl" && this.config.source.crawl) {
+        robotsRules = await fetchRobotsTxt(this.config.source.crawl.baseUrl);
+      }
+      if (robotsRules) {
+        const beforeRobots = filteredSourcePages.length;
+        filteredSourcePages = filteredSourcePages.filter((p) => {
+          const url = normalizeUrlPath(p.url);
+          if (isBlockedByRobots(url, robotsRules)) {
+            this.logger.debug(`Excluding ${url} (blocked by robots.txt)`);
+            return false;
+          }
+          return true;
+        });
+        const robotsExcluded = beforeRobots - filteredSourcePages.length;
+        if (robotsExcluded > 0) {
+          this.logger.info(`Excluded ${robotsExcluded} page${robotsExcluded === 1 ? "" : "s"} by robots.txt`);
+        }
+      }
+    }
+    stageEnd("filter", filterStart);
     const routeStart = stageStart();
     const routePatterns = await buildRoutePatterns(this.cwd);
     stageEnd("route_map", routeStart);
@@ -20452,7 +20737,7 @@ var IndexPipeline = class _IndexPipeline {
     const extractStart = stageStart();
     this.logger.info("Extracting content...");
     const extractedPages = [];
-    for (const sourcePage of sourcePages) {
+    for (const sourcePage of filteredSourcePages) {
       const extracted = sourcePage.html ? extractFromHtml(sourcePage.url, sourcePage.html, this.config) : extractFromMarkdown(sourcePage.url, sourcePage.markdown ?? "", sourcePage.title);
       if (!extracted) {
         this.logger.warn(
@@ -20478,16 +20763,29 @@ var IndexPipeline = class _IndexPipeline {
       seenUrls.add(page.url);
       uniquePages.push(page);
     }
+    const indexablePages = [];
+    for (const page of uniquePages) {
+      const effectiveWeight = page.weight ?? findPageWeight(page.url, this.config.ranking.pageWeights);
+      if (effectiveWeight === 0) {
+        this.logger.debug(`Excluding ${page.url} (zero weight)`);
+        continue;
+      }
+      indexablePages.push(page);
+    }
+    const zeroWeightCount = uniquePages.length - indexablePages.length;
+    if (zeroWeightCount > 0) {
+      this.logger.info(`Excluded ${zeroWeightCount} page${zeroWeightCount === 1 ? "" : "s"} with zero weight`);
+    }
     stageEnd("extract", extractStart);
-    const skippedPages = sourcePages.length - uniquePages.length;
-    this.logger.info(`Extracted ${uniquePages.length} page${uniquePages.length === 1 ? "" : "s"}${skippedPages > 0 ? ` (${skippedPages} skipped)` : ""} (${stageTimingsMs["extract"]}ms)`);
+    const skippedPages = filteredSourcePages.length - indexablePages.length;
+    this.logger.info(`Extracted ${indexablePages.length} page${indexablePages.length === 1 ? "" : "s"}${skippedPages > 0 ? ` (${skippedPages} skipped)` : ""} (${stageTimingsMs["extract"]}ms)`);
     const linkStart = stageStart();
-    const pageSet = new Set(uniquePages.map((page) => normalizeUrlPath(page.url)));
+    const pageSet = new Set(indexablePages.map((page) => normalizeUrlPath(page.url)));
     const incomingLinkCount = /* @__PURE__ */ new Map();
-    for (const page of uniquePages) {
+    for (const page of indexablePages) {
       incomingLinkCount.set(page.url, incomingLinkCount.get(page.url) ?? 0);
     }
-    for (const page of uniquePages) {
+    for (const page of indexablePages) {
       for (const outgoing of page.outgoingLinks) {
         if (!pageSet.has(outgoing)) {
           continue;
@@ -20511,7 +20809,7 @@ var IndexPipeline = class _IndexPipeline {
         });
       }
     }
-    for (const page of uniquePages) {
+    for (const page of indexablePages) {
       const routeMatch = precomputedRoutes.get(normalizeUrlPath(page.url)) ?? mapUrlToRoute(page.url, routePatterns);
       if (routeMatch.routeResolution === "best-effort") {
         if (this.config.source.strictRouteMapping) {
@@ -20797,7 +21095,7 @@ function searchsocketVitePlugin(options = {}) {
         });
         const stats = await pipeline.run({
           changedOnly: options.changedOnly ?? true,
-          force: options.force ?? false,
+          force: (options.force ?? false) || /^(1|true|yes)$/i.test(process.env.SEARCHSOCKET_FORCE_REINDEX ?? ""),
           dryRun: options.dryRun ?? false,
           scopeOverride: options.scope,
           verbose: options.verbose