searchsocket 0.5.0 → 0.6.1

package/dist/index.js

@@ -3,18 +3,20 @@ import path from 'path';
 import { createJiti } from 'jiti';
 import { z } from 'zod';
 import { execSync, spawn } from 'child_process';
-import { createHash } from 'crypto';
+import { FusionAlgorithm, QueryMode } from '@upstash/vector';
+import { timingSafeEqual, createHash } from 'crypto';
 import { load } from 'cheerio';
 import matter from 'gray-matter';
 import fg from 'fast-glob';
 import pLimit from 'p-limit';
-import fs3 from 'fs/promises';
+import fs8 from 'fs/promises';
 import net from 'net';
 import { gunzipSync } from 'zlib';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
 import { createMcpExpressApp } from '@modelcontextprotocol/sdk/server/express.js';
+import { WebStandardStreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js';
 
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __commonJS = (cb, mod) => function __require() {
@@ -5013,32 +5015,32 @@ var require_URL = __commonJS({
           else
             return basepath.substring(0, lastslash + 1) + refpath;
         }
-        function remove_dot_segments(path13) {
-          if (!path13) return path13;
+        function remove_dot_segments(path14) {
+          if (!path14) return path14;
           var output = "";
-          while (path13.length > 0) {
-            if (path13 === "." || path13 === "..") {
-              path13 = "";
+          while (path14.length > 0) {
+            if (path14 === "." || path14 === "..") {
+              path14 = "";
               break;
             }
-            var twochars = path13.substring(0, 2);
-            var threechars = path13.substring(0, 3);
-            var fourchars = path13.substring(0, 4);
+            var twochars = path14.substring(0, 2);
+            var threechars = path14.substring(0, 3);
+            var fourchars = path14.substring(0, 4);
             if (threechars === "../") {
-              path13 = path13.substring(3);
+              path14 = path14.substring(3);
             } else if (twochars === "./") {
-              path13 = path13.substring(2);
+              path14 = path14.substring(2);
             } else if (threechars === "/./") {
-              path13 = "/" + path13.substring(3);
-            } else if (twochars === "/." && path13.length === 2) {
-              path13 = "/";
-            } else if (fourchars === "/../" || threechars === "/.." && path13.length === 3) {
-              path13 = "/" + path13.substring(4);
+              path14 = "/" + path14.substring(3);
+            } else if (twochars === "/." && path14.length === 2) {
+              path14 = "/";
+            } else if (fourchars === "/../" || threechars === "/.." && path14.length === 3) {
+              path14 = "/" + path14.substring(4);
               output = output.replace(/\/?[^\/]*$/, "");
             } else {
-              var segment = path13.match(/(\/?([^\/]*))/)[0];
+              var segment = path14.match(/(\/?([^\/]*))/)[0];
               output += segment;
-              path13 = path13.substring(segment.length);
+              path14 = path14.substring(segment.length);
             }
           }
           return output;
@@ -16634,6 +16636,7 @@ var searchSocketConfigSchema = z.object({
     dropSelectors: z.array(z.string()).optional(),
     ignoreAttr: z.string().optional(),
     noindexAttr: z.string().optional(),
+    imageDescAttr: z.string().optional(),
     respectRobotsNoindex: z.boolean().optional()
   }).optional(),
   transform: z.object({
@@ -16649,35 +16652,48 @@ var searchSocketConfigSchema = z.object({
     headingPathDepth: z.number().int().positive().optional(),
     dontSplitInside: z.array(z.enum(["code", "table", "blockquote"])).optional(),
     prependTitle: z.boolean().optional(),
-    pageSummaryChunk: z.boolean().optional()
+    pageSummaryChunk: z.boolean().optional(),
+    weightHeadings: z.boolean().optional()
   }).optional(),
   upstash: z.object({
     url: z.string().url().optional(),
     token: z.string().min(1).optional(),
     urlEnv: z.string().min(1).optional(),
-    tokenEnv: z.string().min(1).optional()
+    tokenEnv: z.string().min(1).optional(),
+    namespaces: z.object({
+      pages: z.string().min(1).optional(),
+      chunks: z.string().min(1).optional()
+    }).optional()
+  }).optional(),
+  embedding: z.object({
+    model: z.string().optional(),
+    dimensions: z.number().int().positive().optional(),
+    taskType: z.string().optional(),
+    batchSize: z.number().int().positive().optional()
   }).optional(),
   search: z.object({
-    semanticWeight: z.number().min(0).max(1).optional(),
-    inputEnrichment: z.boolean().optional(),
-    reranking: z.boolean().optional(),
     dualSearch: z.boolean().optional(),
     pageSearchWeight: z.number().min(0).max(1).optional()
   }).optional(),
   ranking: z.object({
     enableIncomingLinkBoost: z.boolean().optional(),
     enableDepthBoost: z.boolean().optional(),
+    enableFreshnessBoost: z.boolean().optional(),
+    freshnessDecayRate: z.number().positive().optional(),
+    enableAnchorTextBoost: z.boolean().optional(),
     pageWeights: z.record(z.string(), z.number().min(0)).optional(),
     aggregationCap: z.number().int().positive().optional(),
     aggregationDecay: z.number().min(0).max(1).optional(),
     minChunkScoreRatio: z.number().min(0).max(1).optional(),
-    minScore: z.number().min(0).max(1).optional(),
+    minScoreRatio: z.number().min(0).max(1).optional(),
     scoreGapThreshold: z.number().min(0).max(1).optional(),
     weights: z.object({
       incomingLinks: z.number().optional(),
       depth: z.number().optional(),
       aggregation: z.number().optional(),
-      titleMatch: z.number().optional()
+      titleMatch: z.number().optional(),
+      freshness: z.number().optional(),
+      anchorText: z.number().optional()
     }).optional()
   }).optional(),
   api: z.object({
@@ -16692,12 +16708,28 @@ var searchSocketConfigSchema = z.object({
   }).optional(),
   mcp: z.object({
     enable: z.boolean().optional(),
+    access: z.enum(["public", "private"]).optional(),
     transport: z.enum(["stdio", "http"]).optional(),
     http: z.object({
       port: z.number().int().positive().optional(),
-      path: z.string().optional()
+      path: z.string().optional(),
+      apiKey: z.string().min(1).optional(),
+      apiKeyEnv: z.string().min(1).optional()
+    }).optional(),
+    handle: z.object({
+      path: z.string().optional(),
+      apiKey: z.string().min(1).optional(),
+      enableJsonResponse: z.boolean().optional()
     }).optional()
   }).optional(),
+  llmsTxt: z.object({
+    enable: z.boolean().optional(),
+    outputPath: z.string().optional(),
+    title: z.string().optional(),
+    description: z.string().optional(),
+    generateFull: z.boolean().optional(),
+    serveMarkdownVariants: z.boolean().optional()
+  }).optional(),
   state: z.object({
     dir: z.string().optional()
   }).optional()
@@ -16736,6 +16768,7 @@ function createDefaultConfig(projectId) {
       dropSelectors: DEFAULT_DROP_SELECTORS,
       ignoreAttr: "data-search-ignore",
       noindexAttr: "data-search-noindex",
+      imageDescAttr: "data-search-description",
       respectRobotsNoindex: true
     },
     transform: {
@@ -16745,39 +16778,52 @@ function createDefaultConfig(projectId) {
     },
     chunking: {
       strategy: "hybrid",
-      maxChars: 2200,
+      maxChars: 1500,
       overlapChars: 200,
       minChars: 250,
       headingPathDepth: 3,
       dontSplitInside: ["code", "table", "blockquote"],
       prependTitle: true,
-      pageSummaryChunk: true
+      pageSummaryChunk: true,
+      weightHeadings: true
     },
     upstash: {
-      urlEnv: "UPSTASH_SEARCH_REST_URL",
-      tokenEnv: "UPSTASH_SEARCH_REST_TOKEN"
+      urlEnv: "UPSTASH_VECTOR_REST_URL",
+      tokenEnv: "UPSTASH_VECTOR_REST_TOKEN",
+      namespaces: {
+        pages: "pages",
+        chunks: "chunks"
+      }
+    },
+    embedding: {
+      model: "bge-large-en-v1.5",
+      dimensions: 1024,
+      taskType: "RETRIEVAL_DOCUMENT",
+      batchSize: 100
     },
     search: {
-      semanticWeight: 0.75,
-      inputEnrichment: true,
-      reranking: true,
       dualSearch: true,
       pageSearchWeight: 0.3
     },
     ranking: {
       enableIncomingLinkBoost: true,
       enableDepthBoost: true,
+      enableFreshnessBoost: false,
+      freshnessDecayRate: 1e-3,
+      enableAnchorTextBoost: false,
       pageWeights: {},
       aggregationCap: 5,
       aggregationDecay: 0.5,
       minChunkScoreRatio: 0.5,
-      minScore: 0.3,
+      minScoreRatio: 0.7,
       scoreGapThreshold: 0.4,
       weights: {
         incomingLinks: 0.05,
         depth: 0.03,
         aggregation: 0.1,
-        titleMatch: 0.15
+        titleMatch: 0.15,
+        freshness: 0.1,
+        anchorText: 0.1
       }
     },
     api: {
@@ -16788,12 +16834,23 @@ function createDefaultConfig(projectId) {
     },
     mcp: {
       enable: process.env.NODE_ENV !== "production",
+      access: "private",
       transport: "stdio",
       http: {
         port: 3338,
         path: "/mcp"
+      },
+      handle: {
+        path: "/api/mcp",
+        enableJsonResponse: true
       }
     },
+    llmsTxt: {
+      enable: false,
+      outputPath: "static/llms.txt",
+      generateFull: true,
+      serveMarkdownVariants: false
+    },
     state: {
       dir: ".searchsocket"
     }
@@ -16921,7 +16978,15 @@ ${issues}`
     },
     upstash: {
       ...defaults.upstash,
-      ...parsed.upstash
+      ...parsed.upstash,
+      namespaces: {
+        ...defaults.upstash.namespaces,
+        ...parsed.upstash?.namespaces
+      }
+    },
+    embedding: {
+      ...defaults.embedding,
+      ...parsed.embedding
     },
     search: {
       ...defaults.search,
@@ -16958,8 +17023,16 @@ ${issues}`
       http: {
         ...defaults.mcp.http,
         ...parsed.mcp?.http
+      },
+      handle: {
+        ...defaults.mcp.handle,
+        ...parsed.mcp?.handle
       }
     },
+    llmsTxt: {
+      ...defaults.llmsTxt,
+      ...parsed.llmsTxt
+    },
     state: {
       ...defaults.state,
       ...parsed.state
@@ -16979,6 +17052,15 @@ ${issues}`
       maxDepth: 10
     };
   }
+  if (merged.mcp.access === "public") {
+    const resolvedKey = merged.mcp.http.apiKey ?? (merged.mcp.http.apiKeyEnv ? process.env[merged.mcp.http.apiKeyEnv] : void 0);
+    if (!resolvedKey) {
+      throw new SearchSocketError(
+        "CONFIG_MISSING",
+        '`mcp.access` is "public" but no API key is configured. Set `mcp.http.apiKey` or `mcp.http.apiKeyEnv`.'
+      );
+    }
+  }
   if (merged.source.mode === "crawl" && !merged.source.crawl?.baseUrl) {
     throw new SearchSocketError("CONFIG_MISSING", "`source.crawl.baseUrl` is required when source.mode is crawl.");
   }
@@ -17042,13 +17124,84 @@ function normalizeMarkdown(input) {
 function sanitizeScopeName(scopeName) {
   return scopeName.toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
 }
+function markdownToPlain(markdown) {
+  return markdown.replace(/```[\s\S]*?```/g, " ").replace(/`([^`]+)`/g, "$1").replace(/[#>*_|\-]/g, " ").replace(/\s+/g, " ").trim();
+}
 function toSnippet(markdown, maxLen = 220) {
-  const plain = markdown.replace(/```[\s\S]*?```/g, " ").replace(/`([^`]+)`/g, "$1").replace(/[#>*_|\-]/g, " ").replace(/\s+/g, " ").trim();
+  const plain = markdownToPlain(markdown);
   if (plain.length <= maxLen) {
     return plain;
   }
   return `${plain.slice(0, Math.max(0, maxLen - 1)).trim()}\u2026`;
 }
+function queryAwareExcerpt(markdown, query, maxLen = 220) {
+  const plain = markdownToPlain(markdown);
+  if (plain.length <= maxLen) return plain;
+  const tokens = query.toLowerCase().split(/\s+/).filter((t) => t.length >= 2);
+  if (tokens.length === 0) return toSnippet(markdown, maxLen);
+  const positions = [];
+  for (let ti = 0; ti < tokens.length; ti++) {
+    const escaped = tokens[ti].replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = new RegExp(escaped, "gi");
+    let m;
+    while ((m = re.exec(plain)) !== null) {
+      positions.push({ start: m.index, end: m.index + m[0].length, tokenIdx: ti });
+    }
+  }
+  if (positions.length === 0) return toSnippet(markdown, maxLen);
+  positions.sort((a, b) => a.start - b.start);
+  let bestUniqueCount = 0;
+  let bestTotalCount = 0;
+  let bestLeft = 0;
+  let bestRight = 0;
+  let left = 0;
+  const tokenCounts = /* @__PURE__ */ new Map();
+  for (let right = 0; right < positions.length; right++) {
+    tokenCounts.set(positions[right].tokenIdx, (tokenCounts.get(positions[right].tokenIdx) ?? 0) + 1);
+    while (positions[right].end - positions[left].start > maxLen && left < right) {
+      const leftToken = positions[left].tokenIdx;
+      const cnt = tokenCounts.get(leftToken) - 1;
+      if (cnt === 0) tokenCounts.delete(leftToken);
+      else tokenCounts.set(leftToken, cnt);
+      left++;
+    }
+    const uniqueCount = tokenCounts.size;
+    const totalCount = right - left + 1;
+    if (uniqueCount > bestUniqueCount || uniqueCount === bestUniqueCount && totalCount > bestTotalCount) {
+      bestUniqueCount = uniqueCount;
+      bestTotalCount = totalCount;
+      bestLeft = left;
+      bestRight = right;
+    }
+  }
+  const mid = Math.floor((positions[bestLeft].start + positions[bestRight].end) / 2);
+  let start = Math.max(0, mid - Math.floor(maxLen / 2));
+  let end = Math.min(plain.length, start + maxLen);
+  start = Math.max(0, end - maxLen);
+  if (start > 0) {
+    const spaceIdx = plain.lastIndexOf(" ", start);
+    if (spaceIdx > start - 30) {
+      start = spaceIdx + 1;
+    }
+  }
+  if (end < plain.length) {
+    const spaceIdx = plain.indexOf(" ", end);
+    if (spaceIdx !== -1 && spaceIdx < end + 30) {
+      end = spaceIdx;
+    }
+  }
+  let excerpt = plain.slice(start, end);
+  if (excerpt.length > Math.ceil(maxLen * 1.2)) {
+    excerpt = excerpt.slice(0, maxLen);
+    const lastSpace = excerpt.lastIndexOf(" ");
+    if (lastSpace > maxLen * 0.5) {
+      excerpt = excerpt.slice(0, lastSpace);
+    }
+  }
+  const prefix = start > 0 ? "\u2026" : "";
+  const suffix = end < plain.length ? "\u2026" : "";
+  return `${prefix}${excerpt}${suffix}`;
+}
 function extractFirstParagraph(markdown) {
   const lines = markdown.split("\n");
   let inFence = false;
@@ -17109,162 +17262,342 @@ function ensureStateDirs(cwd, stateDir, scope) {
   fs.mkdirSync(statePath, { recursive: true });
   return { statePath };
 }
-
-// src/vector/upstash.ts
-function chunkIndexName(scope) {
-  return `${scope.projectId}--${scope.scopeName}`;
-}
-function pageIndexName(scope) {
-  return `${scope.projectId}--${scope.scopeName}--pages`;
-}
 var UpstashSearchStore = class {
-  client;
+  index;
+  pagesNs;
+  chunksNs;
   constructor(opts) {
-    this.client = opts.client;
-  }
-  chunkIndex(scope) {
-    return this.client.index(chunkIndexName(scope));
-  }
-  pageIndex(scope) {
-    return this.client.index(pageIndexName(scope));
+    this.index = opts.index;
+    this.pagesNs = opts.index.namespace(opts.pagesNamespace);
+    this.chunksNs = opts.index.namespace(opts.chunksNamespace);
   }
   async upsertChunks(chunks, scope) {
     if (chunks.length === 0) return;
-    const index = this.chunkIndex(scope);
-    const BATCH_SIZE = 100;
+    const BATCH_SIZE = 90;
     for (let i = 0; i < chunks.length; i += BATCH_SIZE) {
       const batch = chunks.slice(i, i + BATCH_SIZE);
-      await index.upsert(batch);
-    }
-  }
-  async search(query, opts, scope) {
-    const index = this.chunkIndex(scope);
-    const results = await index.search({
-      query,
-      limit: opts.limit,
-      semanticWeight: opts.semanticWeight,
-      inputEnrichment: opts.inputEnrichment,
-      reranking: opts.reranking,
-      filter: opts.filter
+      await this.chunksNs.upsert(
+        batch.map((c) => ({
+          id: c.id,
+          data: c.data,
+          metadata: {
+            ...c.metadata,
+            projectId: scope.projectId,
+            scopeName: scope.scopeName,
+            type: c.metadata.type || "chunk"
+          }
+        }))
+      );
+    }
+  }
+  async search(data, opts, scope) {
+    const filterParts = [
+      `projectId = '${scope.projectId}'`,
+      `scopeName = '${scope.scopeName}'`
+    ];
+    if (opts.filter) {
+      filterParts.push(opts.filter);
+    }
+    const results = await this.chunksNs.query({
+      data,
+      topK: opts.limit,
+      includeMetadata: true,
+      filter: filterParts.join(" AND "),
+      queryMode: QueryMode.HYBRID,
+      fusionAlgorithm: FusionAlgorithm.DBSF
+    });
+    return results.map((doc) => ({
+      id: String(doc.id),
+      score: doc.score,
+      metadata: {
+        projectId: doc.metadata?.projectId ?? "",
+        scopeName: doc.metadata?.scopeName ?? "",
+        url: doc.metadata?.url ?? "",
+        path: doc.metadata?.path ?? "",
+        title: doc.metadata?.title ?? "",
+        sectionTitle: doc.metadata?.sectionTitle ?? "",
+        headingPath: doc.metadata?.headingPath ? String(doc.metadata.headingPath).split(" > ").filter(Boolean) : [],
+        snippet: doc.metadata?.snippet ?? "",
+        chunkText: doc.metadata?.chunkText ?? "",
+        ordinal: doc.metadata?.ordinal ?? 0,
+        contentHash: doc.metadata?.contentHash ?? "",
+        depth: doc.metadata?.depth ?? 0,
+        incomingLinks: doc.metadata?.incomingLinks ?? 0,
+        routeFile: doc.metadata?.routeFile ?? "",
+        tags: doc.metadata?.tags ?? [],
+        description: doc.metadata?.description || void 0,
+        keywords: doc.metadata?.keywords?.length ? doc.metadata.keywords : void 0,
+        publishedAt: typeof doc.metadata?.publishedAt === "number" ? doc.metadata.publishedAt : void 0,
+        incomingAnchorText: doc.metadata?.incomingAnchorText || void 0
+      }
+    }));
+  }
+  async searchChunksByUrl(data, url, opts, scope) {
+    const filterParts = [
+      `projectId = '${scope.projectId}'`,
+      `scopeName = '${scope.scopeName}'`,
+      `url = '${url}'`
+    ];
+    if (opts.filter) {
+      filterParts.push(opts.filter);
+    }
+    const results = await this.chunksNs.query({
+      data,
+      topK: opts.limit,
+      includeMetadata: true,
+      filter: filterParts.join(" AND "),
+      queryMode: QueryMode.HYBRID,
+      fusionAlgorithm: FusionAlgorithm.DBSF
     });
     return results.map((doc) => ({
-      id: doc.id,
+      id: String(doc.id),
       score: doc.score,
       metadata: {
         projectId: doc.metadata?.projectId ?? "",
         scopeName: doc.metadata?.scopeName ?? "",
-        url: doc.content.url,
+        url: doc.metadata?.url ?? "",
         path: doc.metadata?.path ?? "",
-        title: doc.content.title,
-        sectionTitle: doc.content.sectionTitle,
-        headingPath: doc.content.headingPath ? doc.content.headingPath.split(" > ").filter(Boolean) : [],
+        title: doc.metadata?.title ?? "",
+        sectionTitle: doc.metadata?.sectionTitle ?? "",
+        headingPath: doc.metadata?.headingPath ? String(doc.metadata.headingPath).split(" > ").filter(Boolean) : [],
         snippet: doc.metadata?.snippet ?? "",
-        chunkText: doc.content.text,
+        chunkText: doc.metadata?.chunkText ?? "",
         ordinal: doc.metadata?.ordinal ?? 0,
         contentHash: doc.metadata?.contentHash ?? "",
         depth: doc.metadata?.depth ?? 0,
         incomingLinks: doc.metadata?.incomingLinks ?? 0,
         routeFile: doc.metadata?.routeFile ?? "",
-        tags: doc.content.tags ? doc.content.tags.split(",").filter(Boolean) : [],
+        tags: doc.metadata?.tags ?? [],
         description: doc.metadata?.description || void 0,
-        keywords: doc.metadata?.keywords ? doc.metadata.keywords.split(",").filter(Boolean) : void 0
+        keywords: doc.metadata?.keywords?.length ? doc.metadata.keywords : void 0,
+        publishedAt: typeof doc.metadata?.publishedAt === "number" ? doc.metadata.publishedAt : void 0,
+        incomingAnchorText: doc.metadata?.incomingAnchorText || void 0
       }
     }));
   }
-  async searchPages(query, opts, scope) {
-    const index = this.pageIndex(scope);
+  async searchPagesByText(data, opts, scope) {
+    return this.queryPages({ data }, opts, scope);
+  }
+  async searchPagesByVector(vector, opts, scope) {
+    return this.queryPages({ vector }, opts, scope);
+  }
+  async queryPages(input, opts, scope) {
+    const filterParts = [
+      `projectId = '${scope.projectId}'`,
+      `scopeName = '${scope.scopeName}'`
+    ];
+    if (opts.filter) {
+      filterParts.push(opts.filter);
+    }
     let results;
     try {
-      results = await index.search({
-        query,
-        limit: opts.limit,
-        semanticWeight: opts.semanticWeight,
-        inputEnrichment: opts.inputEnrichment,
-        reranking: true,
-        filter: opts.filter
+      results = await this.pagesNs.query({
+        ...input,
+        topK: opts.limit,
+        includeMetadata: true,
+        filter: filterParts.join(" AND "),
+        queryMode: QueryMode.HYBRID,
+        fusionAlgorithm: FusionAlgorithm.DBSF
       });
     } catch {
       return [];
     }
     return results.map((doc) => ({
-      id: doc.id,
+      id: String(doc.id),
       score: doc.score,
-      title: doc.content.title,
-      url: doc.content.url,
-      description: doc.content.description ?? "",
-      tags: doc.content.tags ? doc.content.tags.split(",").filter(Boolean) : [],
+      title: doc.metadata?.title ?? "",
+      url: doc.metadata?.url ?? "",
+      description: doc.metadata?.description ?? "",
+      tags: doc.metadata?.tags ?? [],
       depth: doc.metadata?.depth ?? 0,
       incomingLinks: doc.metadata?.incomingLinks ?? 0,
-      routeFile: doc.metadata?.routeFile ?? ""
+      routeFile: doc.metadata?.routeFile ?? "",
+      publishedAt: typeof doc.metadata?.publishedAt === "number" ? doc.metadata.publishedAt : void 0
     }));
   }
-  async deleteByIds(ids, scope) {
+  async deleteByIds(ids, _scope) {
     if (ids.length === 0) return;
-    const index = this.chunkIndex(scope);
-    const BATCH_SIZE = 500;
+    const BATCH_SIZE = 90;
     for (let i = 0; i < ids.length; i += BATCH_SIZE) {
       const batch = ids.slice(i, i + BATCH_SIZE);
-      await index.delete(batch);
+      await this.chunksNs.delete(batch);
     }
   }
   async deleteScope(scope) {
-    try {
-      const chunkIdx = this.chunkIndex(scope);
-      await chunkIdx.deleteIndex();
-    } catch {
-    }
-    try {
-      const pageIdx = this.pageIndex(scope);
-      await pageIdx.deleteIndex();
-    } catch {
+    for (const ns of [this.chunksNs, this.pagesNs]) {
+      const ids = [];
+      let cursor = "0";
+      try {
+        for (; ; ) {
+          const result = await ns.range({
+            cursor,
+            limit: 100,
+            includeMetadata: true
+          });
+          for (const doc of result.vectors) {
+            if (doc.metadata?.projectId === scope.projectId && doc.metadata?.scopeName === scope.scopeName) {
+              ids.push(String(doc.id));
+            }
+          }
+          if (!result.nextCursor || result.nextCursor === "0") break;
+          cursor = result.nextCursor;
+        }
+      } catch {
+      }
+      if (ids.length > 0) {
+        const BATCH_SIZE = 90;
+        for (let i = 0; i < ids.length; i += BATCH_SIZE) {
+          const batch = ids.slice(i, i + BATCH_SIZE);
+          await ns.delete(batch);
+        }
+      }
     }
   }
   async listScopes(projectId) {
-    const allIndexes = await this.client.listIndexes();
-    const prefix = `${projectId}--`;
-    const scopeNames = /* @__PURE__ */ new Set();
-    for (const name of allIndexes) {
-      if (name.startsWith(prefix) && !name.endsWith("--pages")) {
-        const scopeName = name.slice(prefix.length);
-        scopeNames.add(scopeName);
-      }
-    }
-    const scopes = [];
-    for (const scopeName of scopeNames) {
-      const scope = {
-        projectId,
-        scopeName,
-        scopeId: `${projectId}:${scopeName}`
-      };
+    const scopeMap = /* @__PURE__ */ new Map();
+    for (const ns of [this.chunksNs, this.pagesNs]) {
+      let cursor = "0";
+      try {
+        for (; ; ) {
+          const result = await ns.range({
+            cursor,
+            limit: 100,
+            includeMetadata: true
+          });
+          for (const doc of result.vectors) {
+            if (doc.metadata?.projectId === projectId) {
+              const scopeName = doc.metadata.scopeName ?? "";
+              scopeMap.set(scopeName, (scopeMap.get(scopeName) ?? 0) + 1);
+            }
+          }
+          if (!result.nextCursor || result.nextCursor === "0") break;
+          cursor = result.nextCursor;
+        }
+      } catch {
+      }
+    }
+    return [...scopeMap.entries()].map(([scopeName, count]) => ({
+      projectId,
+      scopeName,
+      lastIndexedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      documentCount: count
+    }));
+  }
+  async getContentHashes(scope) {
+    return this.scanHashes(this.chunksNs, scope);
+  }
+  /**
+   * Fetch content hashes for a specific set of chunk keys using direct fetch()
+   * instead of range(). This avoids potential issues with range() returning
+   * vectors from the wrong namespace on hybrid indexes.
+   */
+  async fetchContentHashesForKeys(keys, scope) {
+    const map = /* @__PURE__ */ new Map();
+    if (keys.length === 0) return map;
+    const BATCH_SIZE = 90;
+    for (let i = 0; i < keys.length; i += BATCH_SIZE) {
+      const batch = keys.slice(i, i + BATCH_SIZE);
       try {
-        const info = await this.chunkIndex(scope).info();
-        scopes.push({
-          projectId,
-          scopeName,
-          lastIndexedAt: (/* @__PURE__ */ new Date()).toISOString(),
-          documentCount: info.documentCount
+        const results = await this.chunksNs.fetch(batch, {
+          includeMetadata: true
         });
+        for (const doc of results) {
+          if (doc && doc.metadata?.projectId === scope.projectId && doc.metadata?.scopeName === scope.scopeName && doc.metadata?.contentHash) {
+            map.set(String(doc.id), doc.metadata.contentHash);
+          }
+        }
       } catch {
-        scopes.push({
-          projectId,
-          scopeName,
-          lastIndexedAt: "unknown",
-          documentCount: 0
+      }
+    }
+    return map;
+  }
+  /**
+   * Scan all IDs in the chunks namespace for this scope.
+   * Used for deletion detection (finding stale chunk keys).
+   */
+  async scanChunkIds(scope) {
+    const ids = /* @__PURE__ */ new Set();
+    let cursor = "0";
+    try {
+      for (; ; ) {
+        const result = await this.chunksNs.range({
+          cursor,
+          limit: 100,
+          includeMetadata: true
         });
+        for (const doc of result.vectors) {
+          if (doc.metadata?.projectId === scope.projectId && doc.metadata?.scopeName === scope.scopeName) {
+            ids.add(String(doc.id));
+          }
+        }
+        if (!result.nextCursor || result.nextCursor === "0") break;
+        cursor = result.nextCursor;
       }
+    } catch {
     }
-    return scopes;
+    return ids;
   }
-  async getContentHashes(scope) {
+  async scanHashes(ns, scope) {
+    const map = /* @__PURE__ */ new Map();
+    let cursor = "0";
+    try {
+      for (; ; ) {
+        const result = await ns.range({
+          cursor,
+          limit: 100,
+          includeMetadata: true
+        });
+        for (const doc of result.vectors) {
+          if (doc.metadata?.projectId === scope.projectId && doc.metadata?.scopeName === scope.scopeName && doc.metadata?.contentHash) {
+            map.set(String(doc.id), doc.metadata.contentHash);
+          }
+        }
+        if (!result.nextCursor || result.nextCursor === "0") break;
+        cursor = result.nextCursor;
+      }
+    } catch {
+    }
+    return map;
+  }
+  async listPages(scope, opts) {
+    const cursor = opts?.cursor ?? "0";
+    const limit = opts?.limit ?? 50;
+    try {
+      const result = await this.pagesNs.range({
+        cursor,
+        limit,
+        includeMetadata: true
+      });
+      const pages = result.vectors.filter(
+        (doc) => doc.metadata?.projectId === scope.projectId && doc.metadata?.scopeName === scope.scopeName && (!opts?.pathPrefix || (doc.metadata?.url ?? "").startsWith(opts.pathPrefix))
+      ).map((doc) => ({
+        url: doc.metadata?.url ?? "",
+        title: doc.metadata?.title ?? "",
+        description: doc.metadata?.description ?? "",
+        routeFile: doc.metadata?.routeFile ?? ""
+      }));
+      const response = { pages };
+      if (result.nextCursor && result.nextCursor !== "0") {
+        response.nextCursor = result.nextCursor;
+      }
+      return response;
+    } catch {
+      return { pages: [] };
+    }
+  }
+  async getPageHashes(scope) {
     const map = /* @__PURE__ */ new Map();
-    const index = this.chunkIndex(scope);
     let cursor = "0";
     try {
       for (; ; ) {
-        const result = await index.range({ cursor, limit: 100 });
-        for (const doc of result.documents) {
-          if (doc.metadata?.contentHash) {
-            map.set(doc.id, doc.metadata.contentHash);
+        const result = await this.pagesNs.range({
+          cursor,
+          limit: 100,
+          includeMetadata: true
+        });
+        for (const doc of result.vectors) {
+          if (doc.metadata?.projectId === scope.projectId && doc.metadata?.scopeName === scope.scopeName && doc.metadata?.contentHash) {
+            map.set(String(doc.id), doc.metadata.contentHash);
           }
         }
         if (!result.nextCursor || result.nextCursor === "0") break;
@@ -17274,47 +17607,43 @@ var UpstashSearchStore = class {
     }
     return map;
   }
+  async deletePagesByIds(ids, _scope) {
+    if (ids.length === 0) return;
+    const BATCH_SIZE = 90;
+    for (let i = 0; i < ids.length; i += BATCH_SIZE) {
+      const batch = ids.slice(i, i + BATCH_SIZE);
+      await this.pagesNs.delete(batch);
+    }
+  }
   async upsertPages(pages, scope) {
     if (pages.length === 0) return;
-    const index = this.pageIndex(scope);
-    const BATCH_SIZE = 50;
+    const BATCH_SIZE = 90;
     for (let i = 0; i < pages.length; i += BATCH_SIZE) {
       const batch = pages.slice(i, i + BATCH_SIZE);
-      const docs = batch.map((p) => ({
-        id: p.url,
-        content: {
-          title: p.title,
-          url: p.url,
-          type: "page",
-          description: p.description ?? "",
-          keywords: (p.keywords ?? []).join(","),
-          summary: p.summary ?? "",
-          tags: p.tags.join(",")
-        },
-        metadata: {
-          markdown: p.markdown,
-          projectId: p.projectId,
-          scopeName: p.scopeName,
-          routeFile: p.routeFile,
-          routeResolution: p.routeResolution,
-          incomingLinks: p.incomingLinks,
-          outgoingLinks: p.outgoingLinks,
-          depth: p.depth,
-          indexedAt: p.indexedAt
-        }
-      }));
-      await index.upsert(docs);
+      await this.pagesNs.upsert(
+        batch.map((p) => ({
+          id: p.id,
+          data: p.data,
+          metadata: {
+            ...p.metadata,
+            projectId: scope.projectId,
+            scopeName: scope.scopeName,
+            type: "page"
+          }
+        }))
+      );
     }
   }
   async getPage(url, scope) {
-    const index = this.pageIndex(scope);
     try {
-      const results = await index.fetch([url]);
+      const results = await this.pagesNs.fetch([url], {
+        includeMetadata: true
+      });
       const doc = results[0];
-      if (!doc) return null;
+      if (!doc || !doc.metadata) return null;
       return {
-        url: doc.content.url,
-        title: doc.content.title,
+        url: doc.metadata.url,
+        title: doc.metadata.title,
         markdown: doc.metadata.markdown,
         projectId: doc.metadata.projectId,
         scopeName: doc.metadata.scopeName,
@@ -17322,27 +17651,86 @@ var UpstashSearchStore = class {
         routeResolution: doc.metadata.routeResolution,
         incomingLinks: doc.metadata.incomingLinks,
         outgoingLinks: doc.metadata.outgoingLinks,
+        outgoingLinkUrls: doc.metadata.outgoingLinkUrls ?? void 0,
         depth: doc.metadata.depth,
-        tags: doc.content.tags ? doc.content.tags.split(",").filter(Boolean) : [],
+        tags: doc.metadata.tags ?? [],
         indexedAt: doc.metadata.indexedAt,
-        summary: doc.content.summary || void 0,
-        description: doc.content.description || void 0,
-        keywords: doc.content.keywords ? doc.content.keywords.split(",").filter(Boolean) : void 0
+        summary: doc.metadata.summary || void 0,
+        description: doc.metadata.description || void 0,
+        keywords: doc.metadata.keywords?.length ? doc.metadata.keywords : void 0,
+        publishedAt: typeof doc.metadata.publishedAt === "number" ? doc.metadata.publishedAt : void 0
       };
     } catch {
       return null;
     }
   }
+  async fetchPageWithVector(url, scope) {
+    try {
+      const results = await this.pagesNs.fetch([url], {
+        includeMetadata: true,
+        includeVectors: true
+      });
+      const doc = results[0];
+      if (!doc || !doc.metadata || !doc.vector) return null;
+      if (doc.metadata.projectId !== scope.projectId || doc.metadata.scopeName !== scope.scopeName) {
+        return null;
+      }
+      return { metadata: doc.metadata, vector: doc.vector };
+    } catch {
+      return null;
+    }
+  }
+  async fetchPagesBatch(urls, scope) {
+    if (urls.length === 0) return [];
+    try {
+      const results = await this.pagesNs.fetch(urls, {
+        includeMetadata: true
+      });
+      const out = [];
+      for (const doc of results) {
+        if (!doc || !doc.metadata) continue;
+        if (doc.metadata.projectId !== scope.projectId || doc.metadata.scopeName !== scope.scopeName) {
+          continue;
+        }
+        out.push({
+          url: doc.metadata.url,
+          title: doc.metadata.title,
+          routeFile: doc.metadata.routeFile,
+          outgoingLinkUrls: doc.metadata.outgoingLinkUrls ?? []
+        });
+      }
+      return out;
+    } catch {
+      return [];
+    }
+  }
   async deletePages(scope) {
+    const ids = [];
+    let cursor = "0";
     try {
-      const index = this.pageIndex(scope);
-      await index.reset();
+      for (; ; ) {
+        const result = await this.pagesNs.range({
+          cursor,
+          limit: 100,
+          includeMetadata: true
+        });
+        for (const doc of result.vectors) {
+          if (doc.metadata?.projectId === scope.projectId && doc.metadata?.scopeName === scope.scopeName) {
+            ids.push(String(doc.id));
+          }
+        }
+        if (!result.nextCursor || result.nextCursor === "0") break;
+        cursor = result.nextCursor;
+      }
     } catch {
     }
+    if (ids.length > 0) {
+      await this.deletePagesByIds(ids, scope);
+    }
   }
   async health() {
     try {
-      await this.client.info();
+      await this.index.info();
       return { ok: true };
     } catch (error) {
       return {
@@ -17352,14 +17740,31 @@ var UpstashSearchStore = class {
     }
   }
   async dropAllIndexes(projectId) {
-    const allIndexes = await this.client.listIndexes();
-    const prefix = `${projectId}--`;
-    for (const name of allIndexes) {
-      if (name.startsWith(prefix)) {
-        try {
-          const index = this.client.index(name);
-          await index.deleteIndex();
-        } catch {
+    for (const ns of [this.chunksNs, this.pagesNs]) {
+      const ids = [];
+      let cursor = "0";
+      try {
+        for (; ; ) {
+          const result = await ns.range({
+            cursor,
+            limit: 100,
+            includeMetadata: true
+          });
+          for (const doc of result.vectors) {
+            if (doc.metadata?.projectId === projectId) {
+              ids.push(String(doc.id));
+            }
+          }
+          if (!result.nextCursor || result.nextCursor === "0") break;
+          cursor = result.nextCursor;
+        }
+      } catch {
+      }
+      if (ids.length > 0) {
+        const BATCH_SIZE = 90;
+        for (let i = 0; i < ids.length; i += BATCH_SIZE) {
+          const batch = ids.slice(i, i + BATCH_SIZE);
+          await ns.delete(batch);
         }
       }
     }
@@ -17373,12 +17778,16 @@ async function createUpstashStore(config) {
   if (!url || !token) {
     throw new SearchSocketError(
       "VECTOR_BACKEND_UNAVAILABLE",
-      `Missing Upstash Search credentials. Set ${config.upstash.urlEnv} and ${config.upstash.tokenEnv} environment variables, or pass upstash.url and upstash.token in your config.`
+      `Missing Upstash Vector credentials. Set ${config.upstash.urlEnv} and ${config.upstash.tokenEnv} environment variables, or pass upstash.url and upstash.token in your config.`
     );
   }
-  const { Search } = await import('@upstash/search');
-  const client = new Search({ url, token });
-  return new UpstashSearchStore({ client });
+  const { Index } = await import('@upstash/vector');
+  const index = new Index({ url, token });
+  return new UpstashSearchStore({
+    index,
+    pagesNamespace: config.upstash.namespaces.pages,
+    chunksNamespace: config.upstash.namespaces.chunks
+  });
 }
 function sha1(input) {
   return createHash("sha1").update(input).digest("hex");
@@ -17446,6 +17855,7 @@ function parseHeadingSections(markdown, headingPathDepth) {
     if (normalizeText(current.text)) {
       sections.push({
         sectionTitle: current.sectionTitle,
+        headingLevel: current.headingLevel,
         headingPath: current.headingPath,
         text: current.text.trim()
       });
@@ -17464,6 +17874,7 @@ function parseHeadingSections(markdown, headingPathDepth) {
       headingStack.length = level;
       current = {
         sectionTitle: title,
+        headingLevel: level,
         headingPath: headingStack.filter((entry) => Boolean(entry)).slice(0, headingPathDepth),
         text: `${line}
 `
@@ -17599,6 +18010,7 @@ function splitSection(section, config) {
     return [
       {
         sectionTitle: section.sectionTitle,
+        headingLevel: section.headingLevel,
         headingPath: section.headingPath,
         chunkText: text
       }
@@ -17649,6 +18061,7 @@ ${chunk}`;
   }
   return merged.map((chunkText) => ({
     sectionTitle: section.sectionTitle,
+    headingLevel: section.headingLevel,
     headingPath: section.headingPath,
     chunkText
   }));
@@ -17664,6 +18077,18 @@ function buildSummaryChunkText(page) {
   }
   return parts.join("\n\n");
 }
+function buildEmbeddingTitle(chunk) {
+  if (!chunk.sectionTitle || chunk.headingLevel === void 0) return void 0;
+  if (chunk.headingPath.length > 1) {
+    const path14 = chunk.headingPath.join(" > ");
+    const lastInPath = chunk.headingPath[chunk.headingPath.length - 1];
+    if (lastInPath !== chunk.sectionTitle) {
+      return `${chunk.title} \u2014 ${path14} > ${chunk.sectionTitle}`;
+    }
+    return `${chunk.title} \u2014 ${path14}`;
+  }
+  return `${chunk.title} \u2014 ${chunk.sectionTitle}`;
+}
 function buildEmbeddingText(chunk, prependTitle) {
   if (!prependTitle) return chunk.chunkText;
   const prefix = chunk.sectionTitle ? `${chunk.title} \u2014 ${chunk.sectionTitle}` : chunk.title;
@@ -17694,10 +18119,14 @@ function chunkPage(page, config, scope) {
       tags: page.tags,
       contentHash: "",
       description: page.description,
-      keywords: page.keywords
+      keywords: page.keywords,
+      publishedAt: page.publishedAt,
+      incomingAnchorText: page.incomingAnchorText,
+      meta: page.meta
     };
     const embeddingText = buildEmbeddingText(summaryChunk, config.chunking.prependTitle);
-    summaryChunk.contentHash = sha256(normalizeText(embeddingText));
+    const metaSuffix = page.meta ? JSON.stringify(page.meta, Object.keys(page.meta).sort()) : "";
+    summaryChunk.contentHash = sha256(normalizeText(embeddingText) + metaSuffix);
     chunks.push(summaryChunk);
   }
   const ordinalOffset = config.chunking.pageSummaryChunk ? 1 : 0;
@@ -17714,6 +18143,7 @@ function chunkPage(page, config, scope) {
       path: page.url,
       title: page.title,
       sectionTitle: entry.sectionTitle,
+      headingLevel: entry.headingLevel,
       headingPath: entry.headingPath,
       chunkText: entry.chunkText,
       snippet: toSnippet(entry.chunkText),
@@ -17723,10 +18153,16 @@ function chunkPage(page, config, scope) {
       tags: page.tags,
       contentHash: "",
       description: page.description,
-      keywords: page.keywords
+      keywords: page.keywords,
+      publishedAt: page.publishedAt,
+      incomingAnchorText: page.incomingAnchorText,
+      meta: page.meta
     };
     const embeddingText = buildEmbeddingText(chunk, config.chunking.prependTitle);
-    chunk.contentHash = sha256(normalizeText(embeddingText));
+    const embeddingTitle = config.chunking.weightHeadings ? buildEmbeddingTitle(chunk) : void 0;
+    const chunkMetaSuffix = page.meta ? JSON.stringify(page.meta, Object.keys(page.meta).sort()) : "";
+    const hashInput = embeddingTitle ? `${normalizeText(embeddingText)}|title:${embeddingTitle}` : normalizeText(embeddingText);
+    chunk.contentHash = sha256(hashInput + chunkMetaSuffix);
     chunks.push(chunk);
   }
   return chunks;
@@ -18558,7 +18994,112 @@ function gfm(turndownService) {
   ]);
 }
 
+// src/utils/structured-meta.ts
+var VALID_KEY_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
+function validateMetaKey(key) {
+  return VALID_KEY_RE.test(key);
+}
+function parseMetaValue(content, dataType) {
+  switch (dataType) {
+    case "number": {
+      const n = Number(content);
+      return Number.isFinite(n) ? n : content;
+    }
+    case "boolean":
+      return content === "true";
+    case "string[]":
+      return content ? content.split(",").map((s) => s.trim()) : [];
+    case "date": {
+      const ms = Number(content);
+      return Number.isFinite(ms) ? ms : content;
+    }
+    default:
+      return content;
+  }
+}
+function escapeFilterValue(s) {
+  return s.replace(/'/g, "''");
+}
+function buildMetaFilterString(filters) {
+  const clauses = [];
+  for (const [key, value] of Object.entries(filters)) {
+    if (!validateMetaKey(key)) continue;
+    const field = `meta.${key}`;
+    if (typeof value === "string") {
+      clauses.push(`${field} CONTAINS '${escapeFilterValue(value)}'`);
+    } else if (typeof value === "boolean") {
+      clauses.push(`${field} = ${value}`);
+    } else {
+      clauses.push(`${field} = ${value}`);
+    }
+  }
+  return clauses.join(" AND ");
+}
+
 // src/indexing/extractor.ts
+function normalizeDateToMs(value) {
+  if (value == null) return void 0;
+  if (value instanceof Date) {
+    const ts = value.getTime();
+    return Number.isFinite(ts) ? ts : void 0;
+  }
+  if (typeof value === "string") {
+    const ts = new Date(value).getTime();
+    return Number.isFinite(ts) ? ts : void 0;
+  }
+  if (typeof value === "number") {
+    return Number.isFinite(value) ? value : void 0;
+  }
+  return void 0;
+}
+var FRONTMATTER_DATE_FIELDS = ["date", "publishedAt", "updatedAt", "published_at", "updated_at"];
+function extractPublishedAtFromFrontmatter(data) {
+  for (const field of FRONTMATTER_DATE_FIELDS) {
+    const val = normalizeDateToMs(data[field]);
+    if (val !== void 0) return val;
+  }
+  return void 0;
+}
+function extractPublishedAtFromHtml($) {
+  const jsonLdScripts = $('script[type="application/ld+json"]');
+  for (let i = 0; i < jsonLdScripts.length; i++) {
+    try {
+      const raw = $(jsonLdScripts[i]).html();
+      if (!raw) continue;
+      const parsed = JSON.parse(raw);
+      const candidates = [];
+      if (Array.isArray(parsed)) {
+        candidates.push(...parsed);
+      } else if (parsed && typeof parsed === "object") {
+        candidates.push(parsed);
+        if (Array.isArray(parsed["@graph"])) {
+          candidates.push(...parsed["@graph"]);
+        }
+      }
+      for (const candidate of candidates) {
+        const val = normalizeDateToMs(candidate.datePublished);
+        if (val !== void 0) return val;
+      }
+    } catch {
+    }
+  }
+  const ogTime = $('meta[property="article:published_time"]').attr("content")?.trim();
+  if (ogTime) {
+    const val = normalizeDateToMs(ogTime);
+    if (val !== void 0) return val;
+  }
+  const itempropDate = $('meta[itemprop="datePublished"]').attr("content")?.trim() || $('time[itemprop="datePublished"]').attr("datetime")?.trim();
+  if (itempropDate) {
+    const val = normalizeDateToMs(itempropDate);
+    if (val !== void 0) return val;
+  }
+  const timeEl = $("time[datetime]").first().attr("datetime")?.trim();
+  if (timeEl) {
+    const val = normalizeDateToMs(timeEl);
+    if (val !== void 0) return val;
+  }
+  return void 0;
+}
 function hasTopLevelNoindexComment(markdown) {
   const lines = markdown.split(/\r?\n/);
   let inFence = false;
@@ -18574,6 +19115,97 @@ function hasTopLevelNoindexComment(markdown) {
   }
   return false;
 }
+var GARBAGE_ALT_WORDS = /* @__PURE__ */ new Set([
+  "image",
+  "photo",
+  "picture",
+  "icon",
+  "logo",
+  "banner",
+  "screenshot",
+  "thumbnail",
+  "img",
+  "graphic",
+  "illustration",
+  "spacer",
+  "pixel",
+  "placeholder",
+  "avatar",
+  "background"
+]);
+var IMAGE_EXT_RE = /\.(jpg|jpeg|png|gif|svg|webp|avif|bmp|ico)(\?.*)?$/i;
+function isMeaningfulAlt(alt) {
+  const trimmed = alt.trim();
+  if (!trimmed || trimmed.length < 5) return false;
+  if (IMAGE_EXT_RE.test(trimmed)) return false;
+  if (GARBAGE_ALT_WORDS.has(trimmed.toLowerCase())) return false;
+  return true;
+}
+function resolveImageText(img, $, imageDescAttr) {
+  const imgDesc = img.attr(imageDescAttr)?.trim();
+  if (imgDesc) return imgDesc;
+  const figure = img.closest("figure");
+  if (figure.length) {
+    const figDesc = figure.attr(imageDescAttr)?.trim();
+    if (figDesc) return figDesc;
+  }
+  const alt = img.attr("alt")?.trim() ?? "";
+  const caption = figure.length ? figure.find("figcaption").first().text().trim() : "";
+  if (isMeaningfulAlt(alt) && caption) {
+    return `${alt} \u2014 ${caption}`;
+  }
+  if (isMeaningfulAlt(alt)) {
+    return alt;
+  }
+  if (caption) {
+    return caption;
+  }
+  return null;
+}
+var STOP_ANCHORS = /* @__PURE__ */ new Set([
+  "here",
+  "click",
+  "click here",
+  "read more",
+  "link",
+  "this",
+  "more"
+]);
+function normalizeAnchorText(raw) {
+  const normalized = raw.replace(/\s+/g, " ").trim().toLowerCase();
+  if (normalized.length < 3) return "";
+  if (STOP_ANCHORS.has(normalized)) return "";
+  if (normalized.length > 100) return normalized.slice(0, 100);
+  return normalized;
+}
+function escapeHtml(text) {
+  return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+function preprocessImages(root2, $, imageDescAttr) {
+  root2.find("picture").each((_i, el) => {
+    const picture = $(el);
+    const img = picture.find("img").first();
+    const parentFigure = picture.closest("figure");
+    const text = img.length ? resolveImageText(img, $, imageDescAttr) : null;
+    if (text) {
+      if (parentFigure.length) parentFigure.find("figcaption").remove();
+      picture.replaceWith(`<span>${escapeHtml(text)}</span>`);
+    } else {
+      picture.remove();
+    }
+  });
+  root2.find("img").each((_i, el) => {
+    const img = $(el);
+    const parentFigure = img.closest("figure");
+    const text = resolveImageText(img, $, imageDescAttr);
+    if (text) {
+      if (parentFigure.length) parentFigure.find("figcaption").remove();
+      img.replaceWith(`<span>${escapeHtml(text)}</span>`);
+    } else {
+      img.remove();
+    }
+  });
+}
 function extractFromHtml(url, html, config) {
   const $ = load(html);
   const normalizedUrl = normalizeUrlPath(url);
@@ -18599,6 +19231,20 @@ function extractFromHtml(url, html, config) {
   if (weight === 0) {
     return null;
   }
+  if ($('meta[name="searchsocket:noindex"]').attr("content") === "true") {
+    return null;
+  }
+  const RESERVED_META_KEYS = /* @__PURE__ */ new Set(["noindex", "tags"]);
+  const meta = {};
+  $('meta[name^="searchsocket:"]').each((_i, el) => {
+    const name = $(el).attr("name") ?? "";
+    const key = name.slice("searchsocket:".length);
+    if (!key || RESERVED_META_KEYS.has(key) || !validateMetaKey(key)) return;
+    const content = $(el).attr("content") ?? "";
+    const dataType = $(el).attr("data-type") ?? "string";
+    meta[key] = parseMetaValue(content, dataType);
+  });
+  const componentTags = $('meta[name="searchsocket:tags"]').attr("content")?.trim();
   const description = $("meta[name='description']").attr("content")?.trim() || $("meta[property='og:description']").attr("content")?.trim() || void 0;
   const keywordsRaw = $("meta[name='keywords']").attr("content")?.trim();
   const keywords = keywordsRaw ? keywordsRaw.split(",").map((k) => k.trim()).filter(Boolean) : void 0;
@@ -18610,7 +19256,9 @@ function extractFromHtml(url, html, config) {
     root2.find(selector).remove();
   }
   root2.find(`[${config.extract.ignoreAttr}]`).remove();
+  preprocessImages(root2, $, config.extract.imageDescAttr);
   const outgoingLinks = [];
+  const seenLinkKeys = /* @__PURE__ */ new Set();
   root2.find("a[href]").each((_index, node) => {
     const href = $(node).attr("href");
     if (!href || href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:")) {
@@ -18621,7 +19269,19 @@ function extractFromHtml(url, html, config) {
       if (!["http:", "https:"].includes(parsed.protocol)) {
         return;
       }
-      outgoingLinks.push(normalizeUrlPath(parsed.pathname));
+      const url2 = normalizeUrlPath(parsed.pathname);
+      let anchorText = normalizeAnchorText($(node).text());
+      if (!anchorText) {
+        const imgAlt = $(node).find("img").first().attr("alt") ?? "";
+        if (isMeaningfulAlt(imgAlt)) {
+          anchorText = normalizeAnchorText(imgAlt);
+        }
+      }
+      const key = `${url2}|${anchorText}`;
+      if (!seenLinkKeys.has(key)) {
+        seenLinkKeys.add(key);
+        outgoingLinks.push({ url: url2, anchorText });
+      }
     } catch {
     }
   });
@@ -18646,16 +19306,25 @@ function extractFromHtml(url, html, config) {
     return null;
   }
   const tags = normalizeUrlPath(url).split("/").filter(Boolean).slice(0, 1);
+  const publishedAt = extractPublishedAtFromHtml($);
+  if (componentTags) {
+    const extraTags = componentTags.split(",").map((t) => t.trim()).filter(Boolean);
+    for (const t of extraTags) {
+      if (!tags.includes(t)) tags.push(t);
+    }
+  }
   return {
     url: normalizeUrlPath(url),
     title,
     markdown,
-    outgoingLinks: [...new Set(outgoingLinks)],
+    outgoingLinks,
     noindex: false,
     tags,
     description,
     keywords,
-    weight
+    weight,
+    publishedAt,
+    meta: Object.keys(meta).length > 0 ? meta : void 0
   };
 }
 function extractFromMarkdown(url, markdown, title) {
@@ -18676,6 +19345,24 @@ function extractFromMarkdown(url, markdown, title) {
   if (mdWeight === 0) {
     return null;
   }
+  let mdMeta;
+  const rawMeta = searchsocketMeta?.meta;
+  if (rawMeta && typeof rawMeta === "object" && !Array.isArray(rawMeta)) {
+    const metaObj = {};
+    for (const [key, val] of Object.entries(rawMeta)) {
+      if (!validateMetaKey(key)) continue;
+      if (typeof val === "string" || typeof val === "number" || typeof val === "boolean") {
+        metaObj[key] = val;
+      } else if (Array.isArray(val) && val.every((v) => typeof v === "string")) {
+        metaObj[key] = val;
+      } else if (val instanceof Date) {
+        metaObj[key] = val.getTime();
+      }
+    }
+    if (Object.keys(metaObj).length > 0) {
+      mdMeta = metaObj;
+    }
+  }
   const content = parsed.content;
   const normalized = normalizeMarkdown(content);
   if (!normalizeText(normalized)) {
@@ -18690,6 +19377,7 @@ function extractFromMarkdown(url, markdown, title) {
     fmKeywords = frontmatter.keywords.split(",").map((k) => k.trim()).filter(Boolean);
   }
   if (fmKeywords && fmKeywords.length === 0) fmKeywords = void 0;
+  const publishedAt = extractPublishedAtFromFrontmatter(frontmatter);
   return {
     url: normalizeUrlPath(url),
     title: resolvedTitle,
@@ -18699,7 +19387,9 @@ function extractFromMarkdown(url, markdown, title) {
     tags: normalizeUrlPath(url).split("/").filter(Boolean).slice(0, 1),
     description: fmDescription,
     keywords: fmKeywords,
-    weight: mdWeight
+    weight: mdWeight,
+    publishedAt,
+    meta: mdMeta
   };
 }
 function segmentToRegex(segment) {
@@ -18894,7 +19584,7 @@ async function parseManifest(cwd, outputDir) {
   const manifestPath = path.resolve(cwd, outputDir, "server", "manifest-full.js");
   let content;
   try {
-    content = await fs3.readFile(manifestPath, "utf8");
+    content = await fs8.readFile(manifestPath, "utf8");
   } catch {
     throw new SearchSocketError(
       "BUILD_MANIFEST_NOT_FOUND",
@@ -19205,13 +19895,132 @@ function filePathToUrl(filePath, baseDir) {
   const noExt = relative.replace(/\.md$/i, "").replace(/\/index$/i, "");
   return normalizeUrlPath(noExt || "/");
 }
-function normalizeSvelteToMarkdown(source) {
-  return source.replace(/<script[\s\S]*?<\/script>/g, "").replace(/<style[\s\S]*?<\/style>/g, "").replace(/<[^>]+>/g, " ").replace(/\{[^}]+\}/g, " ").replace(/\s+/g, " ").trim();
+var ROUTE_FILE_RE = /\+(page|layout|error)(@[^.]+)?\.svelte$/;
+function isSvelteComponentFile(filePath) {
+  if (!filePath.endsWith(".svelte")) return false;
+  return !ROUTE_FILE_RE.test(filePath);
 }
-async function loadContentFilesPages(cwd, config, maxPages) {
-  const contentConfig = config.source.contentFiles;
-  if (!contentConfig) {
-    throw new Error("content-files config is missing");
+function extractSvelteComponentMeta(source) {
+  const componentMatch = source.match(/<!--\s*@component\s*([\s\S]*?)\s*-->/);
+  const description = componentMatch?.[1]?.trim() || void 0;
+  const propsMatch = source.match(
+    /let\s+\{([\s\S]*?)\}\s*(?::\s*([^=;{][\s\S]*?))?\s*=\s*\$props\(\)/
+  );
+  const props = [];
+  if (propsMatch) {
+    const destructureBlock = propsMatch[1];
+    const typeAnnotation = propsMatch[2]?.trim();
+    let resolvedTypeMap;
+    if (typeAnnotation && /^[A-Z]\w*$/.test(typeAnnotation)) {
+      resolvedTypeMap = resolveTypeReference(source, typeAnnotation);
+    } else if (typeAnnotation && typeAnnotation.startsWith("{")) {
+      resolvedTypeMap = parseInlineTypeAnnotation(typeAnnotation);
+    }
+    const propEntries = splitDestructureBlock(destructureBlock);
+    for (const entry of propEntries) {
+      const trimmed = entry.trim();
+      if (!trimmed || trimmed.startsWith("...")) continue;
+      let propName;
+      let defaultValue;
+      const renameMatch = trimmed.match(/^(\w+)\s*:\s*\w+\s*(?:=\s*([\s\S]+))?$/);
+      if (renameMatch) {
+        propName = renameMatch[1];
+        defaultValue = renameMatch[2]?.trim();
+      } else {
+        const defaultMatch = trimmed.match(/^(\w+)\s*=\s*([\s\S]+)$/);
+        if (defaultMatch) {
+          propName = defaultMatch[1];
+          defaultValue = defaultMatch[2]?.trim();
+        } else {
+          propName = trimmed.match(/^(\w+)/)?.[1] ?? trimmed;
+        }
+      }
+      const propType = resolvedTypeMap?.get(propName);
+      props.push({
+        name: propName,
+        ...propType ? { type: propType } : {},
+        ...defaultValue ? { default: defaultValue } : {}
+      });
+    }
+  }
+  return { description, props };
+}
+function splitDestructureBlock(block) {
+  const entries = [];
+  let depth = 0;
+  let current = "";
+  for (const ch of block) {
+    if (ch === "{" || ch === "[" || ch === "(") {
+      depth++;
+      current += ch;
+    } else if (ch === "}" || ch === "]" || ch === ")") {
+      depth--;
+      current += ch;
+    } else if (ch === "," && depth === 0) {
+      entries.push(current);
+      current = "";
+    } else {
+      current += ch;
+    }
+  }
+  if (current.trim()) entries.push(current);
+  return entries;
+}
+function resolveTypeReference(source, typeName) {
+  const startRe = new RegExp(`(?:interface\\s+${typeName}\\s*|type\\s+${typeName}\\s*=\\s*)\\{`);
+  const startMatch = source.match(startRe);
+  if (!startMatch || startMatch.index === void 0) return void 0;
+  const bodyStart = startMatch.index + startMatch[0].length;
+  let depth = 1;
+  let i = bodyStart;
+  while (i < source.length && depth > 0) {
+    if (source[i] === "{") depth++;
+    else if (source[i] === "}") depth--;
+    i++;
+  }
+  if (depth !== 0) return void 0;
+  const body = source.slice(bodyStart, i - 1);
+  return parseTypeMembers(body);
+}
+function parseInlineTypeAnnotation(annotation) {
+  const inner = annotation.replace(/^\{/, "").replace(/\}$/, "");
+  return parseTypeMembers(inner);
+}
+function parseTypeMembers(body) {
+  const map = /* @__PURE__ */ new Map();
+  const members = body.split(/[;\n]/).map((m) => m.trim()).filter(Boolean);
+  for (const member of members) {
+    const memberMatch = member.match(/^(\w+)\??\s*:\s*(.+)$/);
+    if (memberMatch) {
+      map.set(memberMatch[1], memberMatch[2].replace(/,\s*$/, "").trim());
+    }
+  }
+  return map;
+}
+function buildComponentMarkdown(componentName, meta) {
+  if (!meta.description && meta.props.length === 0) return "";
+  const parts = [`${componentName} component.`];
+  if (meta.description) {
+    parts.push(meta.description);
+  }
+  if (meta.props.length > 0) {
+    const propEntries = meta.props.map((p) => {
+      let entry = p.name;
+      if (p.type) entry += ` (${p.type})`;
+      if (p.default) entry += ` default: ${p.default}`;
+      return entry;
+    });
+    parts.push(`Props: ${propEntries.join(", ")}.`);
+  }
+  return parts.join(" ");
+}
+function normalizeSvelteToMarkdown(source) {
+  return source.replace(/<script[\s\S]*?<\/script>/g, "").replace(/<style[\s\S]*?<\/style>/g, "").replace(/<[^>]+>/g, " ").replace(/\{[^}]+\}/g, " ").replace(/\s+/g, " ").trim();
+}
+async function loadContentFilesPages(cwd, config, maxPages) {
+  const contentConfig = config.source.contentFiles;
+  if (!contentConfig) {
+    throw new Error("content-files config is missing");
   }
   const baseDir = path.resolve(cwd, contentConfig.baseDir);
   const files = await fg(contentConfig.globs, {
@@ -19223,13 +20032,27 @@ async function loadContentFilesPages(cwd, config, maxPages) {
   const selected = typeof limit === "number" ? files.slice(0, limit) : files;
   const pages = [];
   for (const filePath of selected) {
-    const raw = await fs3.readFile(filePath, "utf8");
-    const markdown = filePath.endsWith(".md") ? raw : normalizeSvelteToMarkdown(raw);
+    const raw = await fs8.readFile(filePath, "utf8");
+    let markdown;
+    let tags;
+    if (filePath.endsWith(".md")) {
+      markdown = raw;
+    } else if (isSvelteComponentFile(filePath)) {
+      const componentName = path.basename(filePath, ".svelte");
+      const meta = extractSvelteComponentMeta(raw);
+      const componentMarkdown = buildComponentMarkdown(componentName, meta);
+      const templateContent = normalizeSvelteToMarkdown(raw);
+      markdown = componentMarkdown ? [componentMarkdown, templateContent].filter(Boolean).join("\n\n") : templateContent;
+      tags = ["component"];
+    } else {
+      markdown = normalizeSvelteToMarkdown(raw);
+    }
     pages.push({
       url: filePathToUrl(filePath, baseDir),
       markdown,
       sourcePath: path.relative(cwd, filePath).replace(/\\/g, "/"),
-      outgoingLinks: []
+      outgoingLinks: [],
+      ...tags ? { tags } : {}
     });
   }
   return pages;
@@ -19359,7 +20182,7 @@ async function loadStaticOutputPages(cwd, config, maxPages) {
   const selected = typeof limit === "number" ? htmlFiles.slice(0, limit) : htmlFiles;
   const pages = [];
   for (const filePath of selected) {
-    const html = await fs3.readFile(filePath, "utf8");
+    const html = await fs8.readFile(filePath, "utf8");
     pages.push({
       url: staticHtmlFileToUrl(filePath, outputDir),
       html,
@@ -19422,7 +20245,7 @@ function isBlockedByRobots(urlPath, rules3) {
 }
 async function loadRobotsTxtFromDir(dir) {
   try {
-    const content = await fs3.readFile(path.join(dir, "robots.txt"), "utf8");
+    const content = await fs8.readFile(path.join(dir, "robots.txt"), "utf8");
     return parseRobotsTxt(content);
   } catch {
     return null;
@@ -19450,29 +20273,65 @@ function nonNegativeOrZero(value) {
 function normalizeForTitleMatch(text) {
   return text.toLowerCase().replace(/[^a-z0-9\s]/g, "").replace(/\s+/g, " ").trim();
 }
-function rankHits(hits, config, query) {
+function rankHits(hits, config, query, debug) {
   const normalizedQuery = query ? normalizeForTitleMatch(query) : "";
   const titleMatchWeight = config.ranking.weights.titleMatch;
   return hits.map((hit) => {
-    let score = Number.isFinite(hit.score) ? hit.score : Number.NEGATIVE_INFINITY;
+    const baseScore = Number.isFinite(hit.score) ? hit.score : Number.NEGATIVE_INFINITY;
+    let score = baseScore;
+    let incomingLinkBoostValue = 0;
     if (config.ranking.enableIncomingLinkBoost) {
       const incomingBoost = Math.log(1 + nonNegativeOrZero(hit.metadata.incomingLinks));
-      score += incomingBoost * config.ranking.weights.incomingLinks;
+      incomingLinkBoostValue = incomingBoost * config.ranking.weights.incomingLinks;
+      score += incomingLinkBoostValue;
     }
+    let depthBoostValue = 0;
     if (config.ranking.enableDepthBoost) {
       const depthBoost = 1 / (1 + nonNegativeOrZero(hit.metadata.depth));
-      score += depthBoost * config.ranking.weights.depth;
+      depthBoostValue = depthBoost * config.ranking.weights.depth;
+      score += depthBoostValue;
     }
+    let titleMatchBoostValue = 0;
     if (normalizedQuery && titleMatchWeight > 0) {
       const normalizedTitle = normalizeForTitleMatch(hit.metadata.title);
       if (normalizedQuery.length > 0 && normalizedTitle.length > 0 && (normalizedTitle.includes(normalizedQuery) || normalizedQuery.includes(normalizedTitle))) {
-        score += titleMatchWeight;
+        titleMatchBoostValue = titleMatchWeight;
+        score += titleMatchBoostValue;
       }
     }
-    return {
+    let freshnessBoostValue = 0;
+    if (config.ranking.enableFreshnessBoost) {
+      const publishedAt = hit.metadata.publishedAt;
+      if (typeof publishedAt === "number" && Number.isFinite(publishedAt)) {
+        const daysSince = Math.max(0, (Date.now() - publishedAt) / 864e5);
+        const decay = 1 / (1 + nonNegativeOrZero(daysSince) * config.ranking.freshnessDecayRate);
+        freshnessBoostValue = decay * config.ranking.weights.freshness;
+        score += freshnessBoostValue;
+      }
+    }
+    let anchorTextMatchBoostValue = 0;
+    if (config.ranking.enableAnchorTextBoost && normalizedQuery && config.ranking.weights.anchorText > 0) {
+      const normalizedAnchorText = normalizeForTitleMatch(hit.metadata.incomingAnchorText ?? "");
+      if (normalizedAnchorText.length > 0 && normalizedQuery.length > 0 && (normalizedAnchorText.includes(normalizedQuery) || normalizedQuery.includes(normalizedAnchorText))) {
+        anchorTextMatchBoostValue = config.ranking.weights.anchorText;
+        score += anchorTextMatchBoostValue;
+      }
+    }
+    const result = {
       hit,
       finalScore: Number.isFinite(score) ? score : Number.NEGATIVE_INFINITY
     };
+    if (debug) {
+      result.breakdown = {
+        baseScore,
+        incomingLinkBoost: incomingLinkBoostValue,
+        depthBoost: depthBoostValue,
+        titleMatchBoost: titleMatchBoostValue,
+        freshnessBoost: freshnessBoostValue,
+        anchorTextMatchBoost: anchorTextMatchBoostValue
+      };
+    }
+    return result;
   }).sort((a, b) => {
     const delta = b.finalScore - a.finalScore;
     return Number.isNaN(delta) ? 0 : delta;
@@ -19481,12 +20340,13 @@ function rankHits(hits, config, query) {
 function trimByScoreGap(results, config) {
   if (results.length === 0) return results;
   const threshold = config.ranking.scoreGapThreshold;
-  const minScore = config.ranking.minScore;
-  if (minScore > 0 && results.length > 0) {
-    const sortedScores = results.map((r) => r.pageScore).sort((a, b) => a - b);
-    const mid = Math.floor(sortedScores.length / 2);
-    const median = sortedScores.length % 2 === 0 ? (sortedScores[mid - 1] + sortedScores[mid]) / 2 : sortedScores[mid];
-    if (median < minScore) return [];
+  const minScoreRatio = config.ranking.minScoreRatio;
+  if (minScoreRatio > 0 && results.length > 0) {
+    const topScore = results[0].pageScore;
+    if (Number.isFinite(topScore) && topScore > 0) {
+      const minThreshold = topScore * minScoreRatio;
+      results = results.filter((r) => r.pageScore >= minThreshold);
+    }
   }
   if (threshold > 0 && results.length > 1) {
     for (let i = 1; i < results.length; i++) {
@@ -19556,61 +20416,99 @@ function aggregateByPage(ranked, config) {
     return Number.isNaN(delta) ? 0 : delta;
   });
 }
-function mergePageAndChunkResults(pageHits, rankedChunks, config) {
-  if (pageHits.length === 0) return rankedChunks;
-  const w = config.search.pageSearchWeight;
-  const pageScoreMap = /* @__PURE__ */ new Map();
-  for (const ph of pageHits) {
-    pageScoreMap.set(ph.url, ph);
-  }
-  const pagesWithChunks = /* @__PURE__ */ new Set();
-  const merged = rankedChunks.map((ranked) => {
-    const url = ranked.hit.metadata.url;
-    const pageHit = pageScoreMap.get(url);
-    if (pageHit) {
-      pagesWithChunks.add(url);
-      const blended = (1 - w) * ranked.finalScore + w * pageHit.score;
-      return {
-        hit: ranked.hit,
-        finalScore: Number.isFinite(blended) ? blended : ranked.finalScore
-      };
+function rankPageHits(pageHits, config, query, debug) {
+  const normalizedQuery = query ? normalizeForTitleMatch(query) : "";
+  const titleMatchWeight = config.ranking.weights.titleMatch;
+  return pageHits.map((hit) => {
+    const baseScore = Number.isFinite(hit.score) ? hit.score : Number.NEGATIVE_INFINITY;
+    let score = baseScore;
+    let incomingLinkBoostValue = 0;
+    if (config.ranking.enableIncomingLinkBoost) {
+      const incomingBoost = Math.log(1 + nonNegativeOrZero(hit.incomingLinks));
+      incomingLinkBoostValue = incomingBoost * config.ranking.weights.incomingLinks;
+      score += incomingLinkBoostValue;
     }
-    return ranked;
-  });
-  for (const [url, pageHit] of pageScoreMap) {
-    if (pagesWithChunks.has(url)) continue;
-    const syntheticScore = pageHit.score * w;
-    const syntheticHit = {
-      id: `page:${url}`,
-      score: pageHit.score,
-      metadata: {
-        projectId: "",
-        scopeName: "",
-        url: pageHit.url,
-        path: pageHit.url,
-        title: pageHit.title,
-        sectionTitle: "",
-        headingPath: [],
-        snippet: pageHit.description || pageHit.title,
-        chunkText: pageHit.description || pageHit.title,
-        ordinal: 0,
-        contentHash: "",
-        depth: pageHit.depth,
-        incomingLinks: pageHit.incomingLinks,
-        routeFile: pageHit.routeFile,
-        tags: pageHit.tags
+    let depthBoostValue = 0;
+    if (config.ranking.enableDepthBoost) {
+      const depthBoost = 1 / (1 + nonNegativeOrZero(hit.depth));
+      depthBoostValue = depthBoost * config.ranking.weights.depth;
+      score += depthBoostValue;
+    }
+    let titleMatchBoostValue = 0;
+    if (normalizedQuery && titleMatchWeight > 0) {
+      const normalizedTitle = normalizeForTitleMatch(hit.title);
+      if (normalizedQuery.length > 0 && normalizedTitle.length > 0 && (normalizedTitle.includes(normalizedQuery) || normalizedQuery.includes(normalizedTitle))) {
+        titleMatchBoostValue = titleMatchWeight;
+        score += titleMatchBoostValue;
+      }
+    }
+    let freshnessBoostValue = 0;
+    if (config.ranking.enableFreshnessBoost) {
+      const publishedAt = hit.publishedAt;
+      if (typeof publishedAt === "number" && Number.isFinite(publishedAt)) {
+        const daysSince = Math.max(0, (Date.now() - publishedAt) / 864e5);
+        const decay = 1 / (1 + nonNegativeOrZero(daysSince) * config.ranking.freshnessDecayRate);
+        freshnessBoostValue = decay * config.ranking.weights.freshness;
+        score += freshnessBoostValue;
       }
+    }
+    const pageWeight = findPageWeight(hit.url, config.ranking.pageWeights);
+    if (pageWeight !== 1) {
+      score *= pageWeight;
+    }
+    const result = {
+      url: hit.url,
+      title: hit.title,
+      description: hit.description,
+      routeFile: hit.routeFile,
+      depth: hit.depth,
+      incomingLinks: hit.incomingLinks,
+      tags: hit.tags,
+      baseScore,
+      finalScore: Number.isFinite(score) ? score : Number.NEGATIVE_INFINITY,
+      publishedAt: hit.publishedAt
     };
-    merged.push({
-      hit: syntheticHit,
-      finalScore: Number.isFinite(syntheticScore) ? syntheticScore : 0
-    });
-  }
-  return merged.sort((a, b) => {
+    if (debug) {
+      result.breakdown = {
+        baseScore,
+        pageWeight,
+        incomingLinkBoost: incomingLinkBoostValue,
+        depthBoost: depthBoostValue,
+        titleMatchBoost: titleMatchBoostValue,
+        freshnessBoost: freshnessBoostValue
+      };
+    }
+    return result;
+  }).filter((p) => findPageWeight(p.url, config.ranking.pageWeights) !== 0).sort((a, b) => {
     const delta = b.finalScore - a.finalScore;
     return Number.isNaN(delta) ? 0 : delta;
   });
 }
+function trimPagesByScoreGap(results, config) {
+  if (results.length === 0) return results;
+  const threshold = config.ranking.scoreGapThreshold;
+  const minScoreRatio = config.ranking.minScoreRatio;
+  if (minScoreRatio > 0 && results.length > 0) {
+    const topScore = results[0].finalScore;
+    if (Number.isFinite(topScore) && topScore > 0) {
+      const minThreshold = topScore * minScoreRatio;
+      results = results.filter((r) => r.finalScore >= minThreshold);
+    }
+  }
+  if (threshold > 0 && results.length > 1) {
+    for (let i = 1; i < results.length; i++) {
+      const prev = results[i - 1].finalScore;
+      const current = results[i].finalScore;
+      if (prev > 0) {
+        const gap = (prev - current) / prev;
+        if (gap >= threshold) {
+          return results.slice(0, i);
+        }
+      }
+    }
+  }
+  return results;
+}
 
 // src/utils/time.ts
 function nowIso() {
@@ -19619,6 +20517,81 @@ function nowIso() {
 function hrTimeMs(start) {
   return Number(process.hrtime.bigint() - start) / 1e6;
 }
+function resolvePageUrl(pageUrl, baseUrl) {
+  if (!baseUrl) return pageUrl;
+  try {
+    return new URL(pageUrl, baseUrl).href;
+  } catch {
+    return pageUrl;
+  }
+}
+function generateLlmsTxt(pages, config) {
+  const title = config.llmsTxt.title ?? config.project.id;
+  const description = config.llmsTxt.description;
+  const baseUrl = config.project.baseUrl;
+  const lines = [`# ${title}`];
+  if (description) {
+    lines.push("", `> ${description}`);
+  }
+  const filtered = pages.filter(
+    (p) => p.url !== "/llms.txt" && p.url !== "/llms-full.txt"
+  );
+  const sorted = [...filtered].sort((a, b) => {
+    if (a.depth !== b.depth) return a.depth - b.depth;
+    return b.incomingLinks - a.incomingLinks;
+  });
+  if (sorted.length > 0) {
+    lines.push("", "## Pages", "");
+    for (const page of sorted) {
+      const url = resolvePageUrl(page.url, baseUrl);
+      if (page.description) {
+        lines.push(`- [${page.title}](${url}): ${page.description}`);
+      } else {
+        lines.push(`- [${page.title}](${url})`);
+      }
+    }
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+function generateLlmsFullTxt(pages, config) {
+  const title = config.llmsTxt.title ?? config.project.id;
+  const description = config.llmsTxt.description;
+  const baseUrl = config.project.baseUrl;
+  const lines = [`# ${title}`];
+  if (description) {
+    lines.push("", `> ${description}`);
+  }
+  const filtered = pages.filter(
+    (p) => p.url !== "/llms.txt" && p.url !== "/llms-full.txt"
+  );
+  const sorted = [...filtered].sort((a, b) => {
+    if (a.depth !== b.depth) return a.depth - b.depth;
+    return b.incomingLinks - a.incomingLinks;
+  });
+  for (const page of sorted) {
+    const url = resolvePageUrl(page.url, baseUrl);
+    lines.push("", "---", "", `## [${page.title}](${url})`, "");
+    lines.push(page.markdown.trim());
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+async function writeLlmsTxt(pages, config, cwd, logger3) {
+  const outputPath = path.resolve(cwd, config.llmsTxt.outputPath);
+  const outputDir = path.dirname(outputPath);
+  await fs8.mkdir(outputDir, { recursive: true });
+  const content = generateLlmsTxt(pages, config);
+  await fs8.writeFile(outputPath, content, "utf8");
+  logger3.info(`Generated llms.txt at ${config.llmsTxt.outputPath}`);
+  if (config.llmsTxt.generateFull) {
+    const fullPath = outputPath.replace(/\.txt$/, "-full.txt");
+    const fullContent = generateLlmsFullTxt(pages, config);
+    await fs8.writeFile(fullPath, fullContent, "utf8");
+    const relativeFull = path.relative(cwd, fullPath);
+    logger3.info(`Generated llms-full.txt at ${relativeFull}`);
+  }
+}
 
 // src/indexing/pipeline.ts
 function buildPageSummary(page, maxChars = 3500) {
@@ -19637,16 +20610,33 @@ function buildPageSummary(page, maxChars = 3500) {
   if (joined.length <= maxChars) return joined;
   return joined.slice(0, maxChars).trim();
 }
+function buildPageContentHash(page) {
+  const parts = [
+    page.title,
+    page.description ?? "",
+    (page.keywords ?? []).slice().sort().join(","),
+    page.tags.slice().sort().join(","),
+    page.markdown,
+    String(page.outgoingLinks),
+    String(page.publishedAt ?? ""),
+    page.incomingAnchorText ?? "",
+    (page.outgoingLinkUrls ?? []).slice().sort().join(","),
+    page.meta ? JSON.stringify(page.meta, Object.keys(page.meta).sort()) : ""
+  ];
+  return sha256(parts.join("|"));
+}
 var IndexPipeline = class _IndexPipeline {
   cwd;
   config;
   store;
   logger;
+  hooks;
   constructor(options) {
     this.cwd = options.cwd;
     this.config = options.config;
     this.store = options.store;
     this.logger = options.logger;
+    this.hooks = options.hooks;
   }
   static async create(options = {}) {
     const cwd = path.resolve(options.cwd ?? process.cwd());
@@ -19656,7 +20646,8 @@ var IndexPipeline = class _IndexPipeline {
       cwd,
       config,
       store,
-      logger: options.logger ?? new Logger()
+      logger: options.logger ?? new Logger(),
+      hooks: options.hooks ?? {}
     });
   }
   getConfig() {
@@ -19677,7 +20668,7 @@ var IndexPipeline = class _IndexPipeline {
     const scope = resolveScope(this.config, options.scopeOverride);
     ensureStateDirs(this.cwd, this.config.state.dir);
     const sourceMode = options.sourceOverride ?? this.config.source.mode;
-    this.logger.info(`Indexing scope "${scope.scopeName}" (source: ${sourceMode}, backend: upstash-search)`);
+    this.logger.info(`Indexing scope "${scope.scopeName}" (source: ${sourceMode}, backend: upstash-vector)`);
     if (options.force) {
       this.logger.info("Force mode enabled \u2014 full rebuild");
     }
@@ -19685,9 +20676,9 @@ var IndexPipeline = class _IndexPipeline {
       this.logger.info("Dry run \u2014 no writes will be performed");
     }
     const manifestStart = stageStart();
-    const existingHashes = options.force ? /* @__PURE__ */ new Map() : await this.store.getContentHashes(scope);
+    const existingPageHashes = options.force ? /* @__PURE__ */ new Map() : await this.store.getPageHashes(scope);
     stageEnd("manifest", manifestStart);
-    this.logger.debug(`Manifest: ${existingHashes.size} existing chunk hashes loaded`);
+    this.logger.debug(`Manifest: ${existingPageHashes.size} existing page hashes loaded`);
     const sourceStart = stageStart();
     this.logger.info(`Loading pages (source: ${sourceMode})...`);
     let sourcePages;
@@ -19764,11 +20755,61 @@ var IndexPipeline = class _IndexPipeline {
         );
         continue;
       }
-      extractedPages.push(extracted);
+      if (sourcePage.tags && sourcePage.tags.length > 0) {
+        extracted.tags = [.../* @__PURE__ */ new Set([...extracted.tags, ...sourcePage.tags])];
+      }
+      let accepted;
+      if (this.hooks.transformPage) {
+        const transformed = await this.hooks.transformPage(extracted);
+        if (transformed === null) {
+          this.logger.debug(`Page ${sourcePage.url} skipped by transformPage hook`);
+          continue;
+        }
+        accepted = transformed;
+      } else {
+        accepted = extracted;
+      }
+      extractedPages.push(accepted);
       this.logger.event("page_extracted", {
-        url: extracted.url
+        url: accepted.url
       });
     }
+    const customRecords = options.customRecords ?? [];
+    if (customRecords.length > 0) {
+      this.logger.info(`Processing ${customRecords.length} custom record${customRecords.length === 1 ? "" : "s"}...`);
+      for (const record of customRecords) {
+        const normalizedUrl = normalizeUrlPath(record.url);
+        const normalized = normalizeMarkdown(record.content);
+        if (!normalized.trim()) {
+          this.logger.warn(`Custom record ${normalizedUrl} has empty content and was skipped.`);
+          continue;
+        }
+        const urlTags = normalizedUrl.split("/").filter(Boolean).slice(0, 1);
+        const tags = record.tags ? [.../* @__PURE__ */ new Set([...urlTags, ...record.tags])] : urlTags;
+        const extracted = {
+          url: normalizedUrl,
+          title: record.title,
+          markdown: normalized,
+          outgoingLinks: [],
+          noindex: false,
+          tags,
+          weight: record.weight
+        };
+        let accepted;
+        if (this.hooks.transformPage) {
+          const transformed = await this.hooks.transformPage(extracted);
+          if (transformed === null) {
+            this.logger.debug(`Custom record ${normalizedUrl} skipped by transformPage hook`);
+            continue;
+          }
+          accepted = transformed;
+        } else {
+          accepted = extracted;
+        }
+        extractedPages.push(accepted);
+        this.logger.event("page_extracted", { url: accepted.url, custom: true });
+      }
+    }
     extractedPages.sort((a, b) => a.url.localeCompare(b.url));
     const uniquePages = [];
     const seenUrls = /* @__PURE__ */ new Set();
@@ -19801,15 +20842,28 @@ var IndexPipeline = class _IndexPipeline {
     const linkStart = stageStart();
     const pageSet = new Set(indexablePages.map((page) => normalizeUrlPath(page.url)));
     const incomingLinkCount = /* @__PURE__ */ new Map();
+    const incomingAnchorTexts = /* @__PURE__ */ new Map();
     for (const page of indexablePages) {
       incomingLinkCount.set(page.url, incomingLinkCount.get(page.url) ?? 0);
     }
     for (const page of indexablePages) {
-      for (const outgoing of page.outgoingLinks) {
+      const seenForCount = /* @__PURE__ */ new Set();
+      const seenForAnchor = /* @__PURE__ */ new Set();
+      for (const { url: outgoing, anchorText } of page.outgoingLinks) {
         if (!pageSet.has(outgoing)) {
           continue;
         }
-        incomingLinkCount.set(outgoing, (incomingLinkCount.get(outgoing) ?? 0) + 1);
+        if (!seenForCount.has(outgoing)) {
+          seenForCount.add(outgoing);
+          incomingLinkCount.set(outgoing, (incomingLinkCount.get(outgoing) ?? 0) + 1);
+        }
+        if (anchorText && !seenForAnchor.has(outgoing)) {
+          seenForAnchor.add(outgoing);
+          if (!incomingAnchorTexts.has(outgoing)) {
+            incomingAnchorTexts.set(outgoing, /* @__PURE__ */ new Set());
+          }
+          incomingAnchorTexts.get(outgoing).add(anchorText);
+        }
       }
     }
     stageEnd("links", linkStart);
@@ -19828,6 +20882,15 @@ var IndexPipeline = class _IndexPipeline {
         });
       }
     }
+    for (const record of customRecords) {
+      const normalizedUrl = normalizeUrlPath(record.url);
+      if (!precomputedRoutes.has(normalizedUrl)) {
+        precomputedRoutes.set(normalizedUrl, {
+          routeFile: "",
+          routeResolution: "exact"
+        });
+      }
+    }
     for (const page of indexablePages) {
       const routeMatch = precomputedRoutes.get(normalizeUrlPath(page.url)) ?? mapUrlToRoute(page.url, routePatterns);
       if (routeMatch.routeResolution === "best-effort") {
@@ -19845,6 +20908,17 @@ var IndexPipeline = class _IndexPipeline {
       } else {
         routeExact += 1;
       }
+      const anchorSet = incomingAnchorTexts.get(page.url);
+      let incomingAnchorText;
+      if (anchorSet && anchorSet.size > 0) {
+        let joined = "";
+        for (const phrase of anchorSet) {
+          const next2 = joined ? `${joined} ${phrase}` : phrase;
+          if (next2.length > 500) break;
+          joined = next2;
+        }
+        incomingAnchorText = joined || void 0;
+      }
       const indexedPage = {
         url: page.url,
         title: page.title,
@@ -19854,40 +20928,113 @@ var IndexPipeline = class _IndexPipeline {
         generatedAt: nowIso(),
         incomingLinks: incomingLinkCount.get(page.url) ?? 0,
         outgoingLinks: page.outgoingLinks.length,
+        outgoingLinkUrls: page.outgoingLinks.map((l) => typeof l === "string" ? l : l.url),
         depth: getUrlDepth(page.url),
         tags: page.tags,
         markdown: page.markdown,
         description: page.description,
-        keywords: page.keywords
+        keywords: page.keywords,
+        publishedAt: page.publishedAt,
+        incomingAnchorText,
+        meta: page.meta
       };
       pages.push(indexedPage);
       this.logger.event("page_indexed", { url: page.url });
     }
+    const pageRecords = pages.map((p) => {
+      const summary = buildPageSummary(p);
+      return {
+        url: p.url,
+        title: p.title,
+        markdown: p.markdown,
+        projectId: scope.projectId,
+        scopeName: scope.scopeName,
+        routeFile: p.routeFile,
+        routeResolution: p.routeResolution,
+        incomingLinks: p.incomingLinks,
+        outgoingLinks: p.outgoingLinks,
+        outgoingLinkUrls: p.outgoingLinkUrls,
+        depth: p.depth,
+        tags: p.tags,
+        indexedAt: p.generatedAt,
+        summary,
+        description: p.description,
+        keywords: p.keywords,
+        contentHash: buildPageContentHash(p),
+        publishedAt: p.publishedAt,
+        meta: p.meta
+      };
+    });
+    const currentPageUrls = new Set(pageRecords.map((r) => r.url));
+    const changedPages = pageRecords.filter(
+      (r) => !existingPageHashes.has(r.url) || existingPageHashes.get(r.url) !== r.contentHash
+    );
+    const deletedPageUrls = [...existingPageHashes.keys()].filter((url) => !currentPageUrls.has(url));
     if (!options.dryRun) {
-      const pageRecords = pages.map((p) => {
-        const summary = buildPageSummary(p);
-        return {
-          url: p.url,
-          title: p.title,
-          markdown: p.markdown,
-          projectId: scope.projectId,
-          scopeName: scope.scopeName,
-          routeFile: p.routeFile,
-          routeResolution: p.routeResolution,
-          incomingLinks: p.incomingLinks,
-          outgoingLinks: p.outgoingLinks,
-          depth: p.depth,
-          tags: p.tags,
-          indexedAt: p.generatedAt,
-          summary,
-          description: p.description,
-          keywords: p.keywords
-        };
-      });
-      await this.store.deletePages(scope);
-      await this.store.upsertPages(pageRecords, scope);
+      if (options.force) {
+        await this.store.deletePages(scope);
+        this.logger.info(`Upserting ${pageRecords.length} page summaries...`);
+        const pageDocs = pageRecords.map((r) => ({
+          id: r.url,
+          data: r.summary ?? r.title,
+          metadata: {
+            title: r.title,
+            url: r.url,
+            description: r.description ?? "",
+            keywords: r.keywords ?? [],
+            summary: r.summary ?? "",
+            tags: r.tags,
+            markdown: r.markdown,
+            routeFile: r.routeFile,
+            routeResolution: r.routeResolution,
+            incomingLinks: r.incomingLinks,
+            outgoingLinks: r.outgoingLinks,
+            outgoingLinkUrls: r.outgoingLinkUrls ?? [],
+            depth: r.depth,
+            indexedAt: r.indexedAt,
+            contentHash: r.contentHash ?? "",
+            publishedAt: r.publishedAt ?? null,
+            ...r.meta && Object.keys(r.meta).length > 0 ? { meta: r.meta } : {}
+          }
+        }));
+        await this.store.upsertPages(pageDocs, scope);
+      } else {
+        if (changedPages.length > 0) {
+          this.logger.info(`Upserting ${changedPages.length} changed page summaries...`);
+          const pageDocs = changedPages.map((r) => ({
+            id: r.url,
+            data: r.summary ?? r.title,
+            metadata: {
+              title: r.title,
+              url: r.url,
+              description: r.description ?? "",
+              keywords: r.keywords ?? [],
+              summary: r.summary ?? "",
+              tags: r.tags,
+              markdown: r.markdown,
+              routeFile: r.routeFile,
+              routeResolution: r.routeResolution,
+              incomingLinks: r.incomingLinks,
+              outgoingLinks: r.outgoingLinks,
+              outgoingLinkUrls: r.outgoingLinkUrls ?? [],
+              depth: r.depth,
+              indexedAt: r.indexedAt,
+              contentHash: r.contentHash ?? "",
+              publishedAt: r.publishedAt ?? null,
+              ...r.meta && Object.keys(r.meta).length > 0 ? { meta: r.meta } : {}
+            }
+          }));
+          await this.store.upsertPages(pageDocs, scope);
+        }
+        if (deletedPageUrls.length > 0) {
+          await this.store.deletePagesByIds(deletedPageUrls, scope);
+        }
+      }
     }
+    const pagesChanged = options.force ? pageRecords.length : changedPages.length;
+    const pagesDeleted = deletedPageUrls.length;
     stageEnd("pages", pagesStart);
+    this.logger.info(`Page changes: ${pagesChanged} changed/new, ${pagesDeleted} deleted, ${pageRecords.length - changedPages.length} unchanged`);
     this.logger.info(`Indexed ${pages.length} page${pages.length === 1 ? "" : "s"} (${routeExact} exact, ${routeBestEffort} best-effort) (${stageTimingsMs["pages"]}ms)`);
     const chunkStart = stageStart();
     this.logger.info("Chunking pages...");
@@ -19896,6 +21043,18 @@ var IndexPipeline = class _IndexPipeline {
     if (typeof maxChunks === "number") {
       chunks = chunks.slice(0, maxChunks);
     }
+    if (this.hooks.transformChunk) {
+      const transformed = [];
+      for (const chunk of chunks) {
+        const result = await this.hooks.transformChunk(chunk);
+        if (result === null) {
+          this.logger.debug(`Chunk ${chunk.chunkKey} skipped by transformChunk hook`);
+          continue;
+        }
+        transformed.push(result);
+      }
+      chunks = transformed;
+    }
     for (const chunk of chunks) {
       this.logger.event("chunked", {
         url: chunk.url,
@@ -19908,7 +21067,12 @@ var IndexPipeline = class _IndexPipeline {
     for (const chunk of chunks) {
       currentChunkMap.set(chunk.chunkKey, chunk);
     }
-    const changedChunks = chunks.filter((chunk) => {
+    const chunkHashStart = stageStart();
+    const currentChunkKeys = chunks.map((c) => c.chunkKey);
+    const existingHashes = options.force ? /* @__PURE__ */ new Map() : await this.store.fetchContentHashesForKeys(currentChunkKeys, scope);
+    stageEnd("chunk_hashes", chunkHashStart);
+    this.logger.debug(`Fetched ${existingHashes.size} existing chunk hashes for ${currentChunkKeys.length} current keys`);
+    let changedChunks = chunks.filter((chunk) => {
       if (options.force) {
         return true;
       }
@@ -19921,37 +21085,45 @@ var IndexPipeline = class _IndexPipeline {
       }
       return existingHash !== chunk.contentHash;
     });
-    const deletes = [...existingHashes.keys()].filter((chunkKey) => !currentChunkMap.has(chunkKey));
+    const existingChunkIds = options.force ? /* @__PURE__ */ new Set() : await this.store.scanChunkIds(scope);
+    const deletes = [...existingChunkIds].filter((chunkKey) => !currentChunkMap.has(chunkKey));
+    if (this.hooks.beforeIndex) {
+      changedChunks = await this.hooks.beforeIndex(changedChunks);
+    }
     this.logger.info(`Changes detected: ${changedChunks.length} changed, ${deletes.length} deleted, ${chunks.length - changedChunks.length} unchanged`);
     const upsertStart = stageStart();
     let documentsUpserted = 0;
     if (!options.dryRun && changedChunks.length > 0) {
-      this.logger.info(`Upserting ${changedChunks.length} chunk${changedChunks.length === 1 ? "" : "s"} to Upstash Search...`);
-      const UPSTASH_CONTENT_LIMIT = 4096;
+      this.logger.info(`Upserting ${changedChunks.length} chunk${changedChunks.length === 1 ? "" : "s"} to Upstash Vector...`);
       const docs = changedChunks.map((chunk) => {
-        const title = chunk.title;
-        const sectionTitle = chunk.sectionTitle ?? "";
-        const url = chunk.url;
-        const tags = chunk.tags.join(",");
-        const headingPath = chunk.headingPath.join(" > ");
-        const otherFieldsLen = title.length + sectionTitle.length + url.length + tags.length + headingPath.length;
-        const textBudget = Math.max(500, UPSTASH_CONTENT_LIMIT - otherFieldsLen - 50);
-        const text = buildEmbeddingText(chunk, this.config.chunking.prependTitle).slice(0, textBudget);
+        const embeddingText = buildEmbeddingText(chunk, this.config.chunking.prependTitle);
+        if (embeddingText.length > 2e3) {
+          this.logger.warn(
+            `Chunk ${chunk.chunkKey} text is ${embeddingText.length} chars (~${Math.round(embeddingText.length / 4)} tokens), which may exceed the 512-token model limit and be silently truncated.`
+          );
+        }
         return {
           id: chunk.chunkKey,
-          content: { title, sectionTitle, text, url, tags, headingPath },
+          data: embeddingText,
           metadata: {
-            projectId: scope.projectId,
-            scopeName: scope.scopeName,
+            url: chunk.url,
             path: chunk.path,
+            title: chunk.title,
+            sectionTitle: chunk.sectionTitle ?? "",
+            headingPath: chunk.headingPath.join(" > "),
             snippet: chunk.snippet,
+            chunkText: embeddingText,
+            tags: chunk.tags,
             ordinal: chunk.ordinal,
             contentHash: chunk.contentHash,
             depth: chunk.depth,
             incomingLinks: chunk.incomingLinks,
             routeFile: chunk.routeFile,
             description: chunk.description ?? "",
-            keywords: (chunk.keywords ?? []).join(",")
+            keywords: chunk.keywords ?? [],
+            publishedAt: chunk.publishedAt ?? null,
+            incomingAnchorText: chunk.incomingAnchorText ?? "",
+            ...chunk.meta && Object.keys(chunk.meta).length > 0 ? { meta: chunk.meta } : {}
           }
         };
       });
@@ -19969,9 +21141,16 @@ var IndexPipeline = class _IndexPipeline {
     } else {
       this.logger.info("No chunks to upsert \u2014 all up to date");
     }
+    if (this.config.llmsTxt.enable && !options.dryRun) {
+      const llmsStart = stageStart();
+      await writeLlmsTxt(pages, this.config, this.cwd, this.logger);
+      stageEnd("llms_txt", llmsStart);
+    }
     this.logger.info("Done.");
-    return {
+    const stats = {
       pagesProcessed: pages.length,
+      pagesChanged,
+      pagesDeleted,
       chunksTotal: chunks.length,
       chunksChanged: changedChunks.length,
       documentsUpserted,
@@ -19980,16 +21159,143 @@ var IndexPipeline = class _IndexPipeline {
       routeBestEffort,
       stageTimingsMs
     };
+    if (this.hooks.afterIndex) {
+      await this.hooks.afterIndex(stats);
+    }
+    return stats;
   }
 };
+
+// src/search/related-pages.ts
+function diceScore(urlA, urlB) {
+  const segmentsA = urlA.split("/").filter(Boolean);
+  const segmentsB = urlB.split("/").filter(Boolean);
+  if (segmentsA.length === 0 && segmentsB.length === 0) return 1;
+  if (segmentsA.length === 0 || segmentsB.length === 0) return 0;
+  let shared = 0;
+  const minLen = Math.min(segmentsA.length, segmentsB.length);
+  for (let i = 0; i < minLen; i++) {
+    if (segmentsA[i] === segmentsB[i]) {
+      shared++;
+    } else {
+      break;
+    }
+  }
+  return 2 * shared / (segmentsA.length + segmentsB.length);
+}
+function compositeScore(isLinked, dice, semantic) {
+  return (isLinked ? 0.5 : 0) + 0.3 * dice + 0.2 * semantic;
+}
+function dominantRelationshipType(isOutgoing, isIncoming, dice) {
+  if (isOutgoing) return "outgoing_link";
+  if (isIncoming) return "incoming_link";
+  if (dice > 0.4) return "sibling";
+  return "semantic";
+}
+
+// src/search/engine.ts
+var rankingOverridesSchema = z.object({
+  ranking: z.object({
+    enableIncomingLinkBoost: z.boolean().optional(),
+    enableDepthBoost: z.boolean().optional(),
+    aggregationCap: z.number().int().positive().optional(),
+    aggregationDecay: z.number().min(0).max(1).optional(),
+    minChunkScoreRatio: z.number().min(0).max(1).optional(),
+    minScoreRatio: z.number().min(0).max(1).optional(),
+    scoreGapThreshold: z.number().min(0).max(1).optional(),
+    weights: z.object({
+      incomingLinks: z.number().optional(),
+      depth: z.number().optional(),
+      aggregation: z.number().optional(),
+      titleMatch: z.number().optional()
+    }).optional()
+  }).optional(),
+  search: z.object({
+    pageSearchWeight: z.number().min(0).max(1).optional()
+  }).optional()
+}).optional();
 var requestSchema = z.object({
   q: z.string().trim().min(1),
   topK: z.number().int().positive().max(100).optional(),
   scope: z.string().optional(),
   pathPrefix: z.string().optional(),
   tags: z.array(z.string()).optional(),
-  groupBy: z.enum(["page", "chunk"]).optional()
+  filters: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(),
+  groupBy: z.enum(["page", "chunk"]).optional(),
+  maxSubResults: z.number().int().positive().max(20).optional(),
+  debug: z.boolean().optional(),
+  rankingOverrides: rankingOverridesSchema
 });
+var MAX_SITE_STRUCTURE_PAGES = 2e3;
+function makeNode(url, depth) {
+  return { url, title: "", depth, routeFile: "", isIndexed: false, childCount: 0, children: [] };
+}
+function buildTree(pages, pathPrefix) {
+  const nodeMap = /* @__PURE__ */ new Map();
+  const root2 = makeNode("/", 0);
+  nodeMap.set("/", root2);
+  for (const page of pages) {
+    const normalized = normalizeUrlPath(page.url);
+    const segments = normalized.split("/").filter(Boolean);
+    if (segments.length === 0) {
+      root2.title = page.title;
+      root2.routeFile = page.routeFile;
+      root2.isIndexed = true;
+      continue;
+    }
+    for (let i = 1; i <= segments.length; i++) {
+      const partialUrl = "/" + segments.slice(0, i).join("/");
+      if (!nodeMap.has(partialUrl)) {
+        nodeMap.set(partialUrl, makeNode(partialUrl, i));
+      }
+    }
+    const node = nodeMap.get(normalized);
+    node.title = page.title;
+    node.routeFile = page.routeFile;
+    node.isIndexed = true;
+  }
+  for (const [url, node] of nodeMap) {
+    if (url === "/") continue;
+    const segments = url.split("/").filter(Boolean);
+    const parentUrl = segments.length === 1 ? "/" : "/" + segments.slice(0, -1).join("/");
+    const parent = nodeMap.get(parentUrl) ?? root2;
+    parent.children.push(node);
+  }
+  const sortAndCount = (node) => {
+    node.children.sort((a, b) => a.url.localeCompare(b.url));
+    node.childCount = node.children.length;
+    for (const child of node.children) {
+      sortAndCount(child);
+    }
+  };
+  sortAndCount(root2);
+  if (pathPrefix) {
+    const normalizedPrefix = normalizeUrlPath(pathPrefix);
+    const subtreeRoot = nodeMap.get(normalizedPrefix);
+    if (subtreeRoot) {
+      return subtreeRoot;
+    }
+    return makeNode(normalizedPrefix, normalizedPrefix.split("/").filter(Boolean).length);
+  }
+  return root2;
+}
+function mergeRankingOverrides(base, overrides) {
+  return {
+    ...base,
+    search: {
+      ...base.search,
+      ...overrides.search
+    },
+    ranking: {
+      ...base.ranking,
+      ...overrides.ranking,
+      weights: {
+        ...base.ranking.weights,
+        ...overrides.ranking?.weights
+      }
+    }
+  };
+}
 var SearchEngine = class _SearchEngine {
   cwd;
   config;
@@ -20019,125 +21325,203 @@ var SearchEngine = class _SearchEngine {
     }
     const input = parsed.data;
     const totalStart = process.hrtime.bigint();
+    const effectiveConfig = input.debug && input.rankingOverrides ? mergeRankingOverrides(this.config, input.rankingOverrides) : this.config;
     const resolvedScope = resolveScope(this.config, input.scope);
     const topK = input.topK ?? 10;
+    const maxSubResults = input.maxSubResults ?? 5;
     const groupByPage = (input.groupBy ?? "page") === "page";
-    const candidateK = groupByPage ? Math.max(topK * 10, 50) : Math.max(50, topK);
-    const filterParts = [];
-    if (input.pathPrefix) {
-      const prefix = input.pathPrefix.startsWith("/") ? input.pathPrefix : `/${input.pathPrefix}`;
-      filterParts.push(`url GLOB '${prefix}*'`);
-    }
-    if (input.tags && input.tags.length > 0) {
-      for (const tag of input.tags) {
-        filterParts.push(`tags GLOB '*${tag}*'`);
+    const queryText = input.q;
+    const pathPrefix = input.pathPrefix ? input.pathPrefix.startsWith("/") ? input.pathPrefix : `/${input.pathPrefix}` : void 0;
+    const filterTags = input.tags && input.tags.length > 0 ? input.tags : void 0;
+    const metaFilterStr = input.filters && Object.keys(input.filters).length > 0 ? buildMetaFilterString(input.filters) : "";
+    const metaFilter = metaFilterStr || void 0;
+    const applyPagePostFilters = (hits) => {
+      let filtered = hits;
+      if (pathPrefix) {
+        filtered = filtered.filter((h) => h.url.startsWith(pathPrefix));
+      }
+      if (filterTags) {
+        filtered = filtered.filter(
+          (h) => filterTags.every((tag) => h.tags.includes(tag))
+        );
       }
-    }
-    const filter = filterParts.length > 0 ? filterParts.join(" AND ") : void 0;
-    const useDualSearch = this.config.search.dualSearch && groupByPage;
+      return filtered;
+    };
+    const applyChunkPostFilters = (hits) => {
+      let filtered = hits;
+      if (filterTags) {
+        filtered = filtered.filter(
+          (h) => filterTags.every((tag) => h.metadata.tags.includes(tag))
+        );
+      }
+      return filtered;
+    };
     const searchStart = process.hrtime.bigint();
-    let ranked;
-    if (useDualSearch) {
-      const chunkLimit = Math.max(topK * 10, 100);
-      const pageLimit = 20;
-      const [pageHits, chunkHits] = await Promise.all([
-        this.store.searchPages(
-          input.q,
-          {
-            limit: pageLimit,
-            semanticWeight: this.config.search.semanticWeight,
-            inputEnrichment: this.config.search.inputEnrichment,
-            filter
-          },
-          resolvedScope
-        ),
-        this.store.search(
-          input.q,
-          {
-            limit: chunkLimit,
-            semanticWeight: this.config.search.semanticWeight,
-            inputEnrichment: this.config.search.inputEnrichment,
-            reranking: false,
-            filter
-          },
+    if (groupByPage) {
+      const fetchMultiplier = pathPrefix || filterTags ? 2 : 1;
+      const pageLimit = Math.max(topK * 2, 20);
+      const pageHits = await this.store.searchPagesByText(
+        queryText,
+        { limit: pageLimit * fetchMultiplier, filter: metaFilter },
+        resolvedScope
+      );
+      const filteredPages = applyPagePostFilters(pageHits);
+      let rankedPages = rankPageHits(filteredPages, effectiveConfig, input.q, input.debug);
+      rankedPages = trimPagesByScoreGap(rankedPages, effectiveConfig);
+      const topPages = rankedPages.slice(0, topK);
+      const chunkPromises = topPages.map(
+        (page) => this.store.searchChunksByUrl(
+          queryText,
+          page.url,
+          { limit: maxSubResults, filter: metaFilter },
           resolvedScope
-        )
-      ]);
-      const rankedChunks = rankHits(chunkHits, this.config, input.q);
-      ranked = mergePageAndChunkResults(pageHits, rankedChunks, this.config);
+        ).then((chunks) => applyChunkPostFilters(chunks))
+      );
+      const allChunks = await Promise.all(chunkPromises);
+      const searchMs = hrTimeMs(searchStart);
+      const results = this.buildPageFirstResults(topPages, allChunks, input.q, input.debug, maxSubResults);
+      return {
+        q: input.q,
+        scope: resolvedScope.scopeName,
+        results,
+        meta: {
+          timingsMs: {
+            search: Math.round(searchMs),
+            total: Math.round(hrTimeMs(totalStart))
+          }
+        }
+      };
     } else {
+      const candidateK = Math.max(50, topK);
+      const fetchMultiplier = pathPrefix || filterTags ? 2 : 1;
       const hits = await this.store.search(
-        input.q,
-        {
-          limit: candidateK,
-          semanticWeight: this.config.search.semanticWeight,
-          inputEnrichment: this.config.search.inputEnrichment,
-          reranking: this.config.search.reranking,
-          filter
-        },
+        queryText,
+        { limit: candidateK * fetchMultiplier, filter: metaFilter },
         resolvedScope
       );
-      ranked = rankHits(hits, this.config, input.q);
-    }
-    const searchMs = hrTimeMs(searchStart);
-    const results = this.buildResults(ranked, topK, groupByPage, input.q);
-    return {
-      q: input.q,
-      scope: resolvedScope.scopeName,
-      results,
-      meta: {
-        timingsMs: {
-          search: Math.round(searchMs),
-          total: Math.round(hrTimeMs(totalStart))
+      let filtered = hits;
+      if (pathPrefix) {
+        filtered = filtered.filter((h) => h.metadata.url.startsWith(pathPrefix));
+      }
+      if (filterTags) {
+        filtered = filtered.filter(
+          (h) => filterTags.every((tag) => h.metadata.tags.includes(tag))
+        );
+      }
+      const ranked = rankHits(filtered, effectiveConfig, input.q, input.debug);
+      const searchMs = hrTimeMs(searchStart);
+      const results = this.buildResults(ranked, topK, false, maxSubResults, input.q, input.debug, effectiveConfig);
+      return {
+        q: input.q,
+        scope: resolvedScope.scopeName,
+        results,
+        meta: {
+          timingsMs: {
+            search: Math.round(searchMs),
+            total: Math.round(hrTimeMs(totalStart))
+          }
         }
+      };
+    }
+  }
+  buildPageFirstResults(rankedPages, allChunks, query, debug, maxSubResults = 5) {
+    return rankedPages.map((page, i) => {
+      const chunks = allChunks[i] ?? [];
+      const bestChunk = chunks[0];
+      const snippet = bestChunk ? query ? queryAwareExcerpt(bestChunk.metadata.chunkText, query) : toSnippet(bestChunk.metadata.chunkText) : page.description || page.title;
+      const result = {
+        url: page.url,
+        title: page.title,
+        sectionTitle: bestChunk?.metadata.sectionTitle || void 0,
+        snippet,
+        chunkText: bestChunk?.metadata.chunkText || void 0,
+        score: Number(page.finalScore.toFixed(6)),
+        routeFile: page.routeFile,
+        chunks: chunks.length > 0 ? chunks.slice(0, maxSubResults).map((c) => ({
+          sectionTitle: c.metadata.sectionTitle || void 0,
+          snippet: query ? queryAwareExcerpt(c.metadata.chunkText, query) : toSnippet(c.metadata.chunkText),
+          chunkText: c.metadata.chunkText || void 0,
+          headingPath: c.metadata.headingPath,
+          score: Number(c.score.toFixed(6))
+        })) : void 0
+      };
+      if (debug && page.breakdown) {
+        result.breakdown = {
+          baseScore: page.breakdown.baseScore,
+          incomingLinkBoost: page.breakdown.incomingLinkBoost,
+          depthBoost: page.breakdown.depthBoost,
+          titleMatchBoost: page.breakdown.titleMatchBoost,
+          freshnessBoost: page.breakdown.freshnessBoost,
+          anchorTextMatchBoost: 0
+        };
       }
-    };
+      return result;
+    });
   }
-  ensureSnippet(hit) {
+  ensureSnippet(hit, query) {
+    const chunkText = hit.hit.metadata.chunkText;
+    if (query && chunkText) return queryAwareExcerpt(chunkText, query);
     const snippet = hit.hit.metadata.snippet;
     if (snippet && snippet.length >= 30) return snippet;
-    const chunkText = hit.hit.metadata.chunkText;
     if (chunkText) return toSnippet(chunkText);
     return snippet || "";
   }
-  buildResults(ordered, topK, groupByPage, _query) {
+  buildResults(ordered, topK, groupByPage, maxSubResults, query, debug, config) {
+    const cfg = config ?? this.config;
     if (groupByPage) {
-      let pages = aggregateByPage(ordered, this.config);
-      pages = trimByScoreGap(pages, this.config);
-      const minRatio = this.config.ranking.minChunkScoreRatio;
+      let pages = aggregateByPage(ordered, cfg);
+      pages = trimByScoreGap(pages, cfg);
+      const minRatio = cfg.ranking.minChunkScoreRatio;
       return pages.slice(0, topK).map((page) => {
         const bestScore = page.bestChunk.finalScore;
         const minChunkScore = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
-        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minChunkScore).slice(0, 5);
-        return {
+        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minChunkScore).slice(0, maxSubResults);
+        const result = {
           url: page.url,
           title: page.title,
           sectionTitle: page.bestChunk.hit.metadata.sectionTitle || void 0,
-          snippet: this.ensureSnippet(page.bestChunk),
+          snippet: this.ensureSnippet(page.bestChunk, query),
+          chunkText: page.bestChunk.hit.metadata.chunkText || void 0,
           score: Number(page.pageScore.toFixed(6)),
           routeFile: page.routeFile,
-          chunks: meaningful.length > 1 ? meaningful.map((c) => ({
+          chunks: meaningful.length >= 1 ? meaningful.map((c) => ({
             sectionTitle: c.hit.metadata.sectionTitle || void 0,
-            snippet: this.ensureSnippet(c),
+            snippet: this.ensureSnippet(c, query),
+            chunkText: c.hit.metadata.chunkText || void 0,
             headingPath: c.hit.metadata.headingPath,
             score: Number(c.finalScore.toFixed(6))
           })) : void 0
         };
+        if (debug && page.bestChunk.breakdown) {
+          result.breakdown = page.bestChunk.breakdown;
+        }
+        return result;
       });
     } else {
       let filtered = ordered;
-      const minScore = this.config.ranking.minScore;
-      if (minScore > 0) {
-        filtered = ordered.filter((entry) => entry.finalScore >= minScore);
-      }
-      return filtered.slice(0, topK).map(({ hit, finalScore }) => ({
-        url: hit.metadata.url,
-        title: hit.metadata.title,
-        sectionTitle: hit.metadata.sectionTitle || void 0,
-        snippet: this.ensureSnippet({ hit, finalScore }),
-        score: Number(finalScore.toFixed(6)),
-        routeFile: hit.metadata.routeFile
-      }));
+      const minScoreRatio = cfg.ranking.minScoreRatio;
+      if (minScoreRatio > 0 && ordered.length > 0) {
+        const topScore = ordered[0].finalScore;
+        if (Number.isFinite(topScore) && topScore > 0) {
+          const threshold = topScore * minScoreRatio;
+          filtered = ordered.filter((entry) => entry.finalScore >= threshold);
+        }
+      }
+      return filtered.slice(0, topK).map(({ hit, finalScore, breakdown }) => {
+        const result = {
+          url: hit.metadata.url,
+          title: hit.metadata.title,
+          sectionTitle: hit.metadata.sectionTitle || void 0,
+          snippet: this.ensureSnippet({ hit, finalScore }, query),
+          chunkText: hit.metadata.chunkText || void 0,
+          score: Number(finalScore.toFixed(6)),
+          routeFile: hit.metadata.routeFile
+        };
+        if (debug && breakdown) {
+          result.breakdown = breakdown;
+        }
+        return result;
+      });
     }
   }
   async getPage(pathOrUrl, scope) {
@@ -20163,6 +21547,116 @@ var SearchEngine = class _SearchEngine {
       markdown: page.markdown
     };
   }
+  async listPages(opts) {
+    const resolvedScope = resolveScope(this.config, opts?.scope);
+    const pathPrefix = opts?.pathPrefix ? opts.pathPrefix.startsWith("/") ? opts.pathPrefix : `/${opts.pathPrefix}` : void 0;
+    return this.store.listPages(resolvedScope, {
+      cursor: opts?.cursor,
+      limit: opts?.limit,
+      pathPrefix
+    });
+  }
+  async getSiteStructure(opts) {
+    const maxPages = Math.min(opts?.maxPages ?? MAX_SITE_STRUCTURE_PAGES, MAX_SITE_STRUCTURE_PAGES);
+    const allPages = [];
+    let cursor;
+    let truncated = false;
+    do {
+      const result = await this.listPages({
+        pathPrefix: opts?.pathPrefix,
+        scope: opts?.scope,
+        cursor,
+        limit: 200
+      });
+      allPages.push(...result.pages);
+      cursor = result.nextCursor;
+      if (allPages.length >= maxPages) {
+        truncated = allPages.length > maxPages || !!cursor;
+        allPages.length = maxPages;
+        break;
+      }
+    } while (cursor);
+    const root2 = buildTree(allPages, opts?.pathPrefix);
+    return {
+      root: root2,
+      totalPages: allPages.length,
+      truncated
+    };
+  }
+  async getRelatedPages(pathOrUrl, opts) {
+    const resolvedScope = resolveScope(this.config, opts?.scope);
+    const urlPath = this.resolveInputPath(pathOrUrl);
+    const topK = Math.min(opts?.topK ?? 10, 25);
+    const source = await this.store.fetchPageWithVector(urlPath, resolvedScope);
+    if (!source) {
+      throw new SearchSocketError("INVALID_REQUEST", `Indexed page not found for ${urlPath}`, 404);
+    }
+    const sourceOutgoing = new Set(source.metadata.outgoingLinkUrls ?? []);
+    const semanticHits = await this.store.searchPagesByVector(
+      source.vector,
+      { limit: 50 },
+      resolvedScope
+    );
+    const filteredHits = semanticHits.filter((h) => h.url !== urlPath);
+    const semanticScoreMap = /* @__PURE__ */ new Map();
+    for (const hit of filteredHits) {
+      semanticScoreMap.set(hit.url, hit.score);
+    }
+    const candidateUrls = /* @__PURE__ */ new Set();
+    for (const hit of filteredHits) {
+      candidateUrls.add(hit.url);
+    }
+    for (const url of sourceOutgoing) {
+      if (url !== urlPath) candidateUrls.add(url);
+    }
+    const missingUrls = [...sourceOutgoing].filter(
+      (u) => u !== urlPath && !semanticScoreMap.has(u)
+    );
+    const fetchedPages = missingUrls.length > 0 ? await this.store.fetchPagesBatch(missingUrls, resolvedScope) : [];
+    const metaMap = /* @__PURE__ */ new Map();
+    for (const hit of filteredHits) {
+      metaMap.set(hit.url, { title: hit.title, routeFile: hit.routeFile, outgoingLinkUrls: [] });
+    }
+    for (const p of fetchedPages) {
+      metaMap.set(p.url, { title: p.title, routeFile: p.routeFile, outgoingLinkUrls: p.outgoingLinkUrls });
+    }
+    const semanticUrls = filteredHits.map((h) => h.url);
+    if (semanticUrls.length > 0) {
+      const semanticPageData = await this.store.fetchPagesBatch(semanticUrls, resolvedScope);
+      for (const p of semanticPageData) {
+        const existing = metaMap.get(p.url);
+        if (existing) {
+          existing.outgoingLinkUrls = p.outgoingLinkUrls;
+        }
+      }
+    }
+    const candidates = [];
+    for (const url of candidateUrls) {
+      const meta = metaMap.get(url);
+      if (!meta) continue;
+      const isOutgoing = sourceOutgoing.has(url);
+      const isIncoming = meta.outgoingLinkUrls.includes(urlPath);
+      const isLinked = isOutgoing || isIncoming;
+      const dice = diceScore(urlPath, url);
+      const semantic = semanticScoreMap.get(url) ?? 0;
+      const score = compositeScore(isLinked, dice, semantic);
+      const relationshipType = dominantRelationshipType(isOutgoing, isIncoming, dice);
+      candidates.push({
+        url,
+        title: meta.title,
+        score: Number(score.toFixed(6)),
+        relationshipType,
+        routeFile: meta.routeFile
+      });
+    }
+    candidates.sort((a, b) => b.score - a.score);
+    const results = candidates.slice(0, topK);
+    return {
+      sourceUrl: urlPath,
+      scope: resolvedScope.scopeName,
+      relatedPages: results
+    };
+  }
   async health() {
     return this.store.health();
   }
@@ -20185,14 +21679,40 @@ function createServer(engine) {
   server.registerTool(
     "search",
     {
-      description: "Semantic site search powered by Upstash Search. Returns url/title/snippet/score/routeFile for each match. Supports optional scope, pathPrefix, tags, topK, and groupBy.",
+      description: `Semantic site search powered by Upstash Search. Returns url, title, snippet, chunkText, score, and routeFile per result. chunkText contains the full raw chunk markdown. When groupBy is 'page' (default), each result includes a chunks array with section-level sub-results containing sectionTitle, headingPath, snippet, and score. Supports optional filters for structured metadata (e.g. {"version": 2, "deprecated": false}).`,
       inputSchema: {
         query: z.string().min(1),
         scope: z.string().optional(),
         topK: z.number().int().positive().max(100).optional(),
         pathPrefix: z.string().optional(),
         tags: z.array(z.string()).optional(),
-        groupBy: z.enum(["page", "chunk"]).optional()
+        filters: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(),
+        groupBy: z.enum(["page", "chunk"]).optional(),
+        maxSubResults: z.number().int().positive().max(20).optional()
+      },
+      outputSchema: {
+        q: z.string(),
+        scope: z.string(),
+        results: z.array(z.object({
+          url: z.string(),
+          title: z.string(),
+          sectionTitle: z.string().optional(),
+          snippet: z.string(),
+          score: z.number(),
+          routeFile: z.string(),
+          chunks: z.array(z.object({
+            sectionTitle: z.string().optional(),
+            snippet: z.string(),
+            headingPath: z.array(z.string()),
+            score: z.number()
+          })).optional()
+        })),
+        meta: z.object({
+          timingsMs: z.object({
+            search: z.number(),
+            total: z.number()
+          })
+        })
       }
     },
     async (input) => {
@@ -20202,7 +21722,9 @@ function createServer(engine) {
         scope: input.scope,
         pathPrefix: input.pathPrefix,
         tags: input.tags,
-        groupBy: input.groupBy
+        filters: input.filters,
+        groupBy: input.groupBy,
+        maxSubResults: input.maxSubResults
       });
       return {
         content: [
@@ -20210,7 +21732,8 @@ function createServer(engine) {
             type: "text",
             text: JSON.stringify(result, null, 2)
           }
-        ]
+        ],
+        structuredContent: result
       };
     }
   );
@@ -20235,8 +21758,134 @@ function createServer(engine) {
       };
     }
   );
+  server.registerTool(
+    "list_pages",
+    {
+      description: "List indexed pages with optional path prefix filtering and cursor-based pagination. Returns url, title, description, and routeFile for each page. Use nextCursor to fetch subsequent pages.",
+      inputSchema: {
+        pathPrefix: z.string().optional(),
+        cursor: z.string().optional(),
+        limit: z.number().int().positive().max(200).optional(),
+        scope: z.string().optional()
+      }
+    },
+    async (input) => {
+      const result = await engine.listPages({
+        pathPrefix: input.pathPrefix,
+        cursor: input.cursor,
+        limit: input.limit,
+        scope: input.scope
+      });
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify(result, null, 2)
+          }
+        ]
+      };
+    }
+  );
+  server.registerTool(
+    "get_site_structure",
+    {
+      description: "Returns the hierarchical page tree derived from URL paths. Use this to understand site navigation structure, find where pages belong, or scope further operations to a section. Nodes with isIndexed: false are implicit structural parents not directly in the index. Large sites (>2000 pages) return truncated: true.",
+      inputSchema: {
+        pathPrefix: z.string().optional(),
+        scope: z.string().optional(),
+        maxPages: z.number().int().positive().max(2e3).optional()
+      }
+    },
+    async (input) => {
+      const result = await engine.getSiteStructure({
+        pathPrefix: input.pathPrefix,
+        scope: input.scope,
+        maxPages: input.maxPages
+      });
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify(result, null, 2)
+          }
+        ]
+      };
+    }
+  );
+  server.registerTool(
+    "find_source_file",
+    {
+      description: "Find the SvelteKit source file for a piece of site content. Use this when you need to locate and edit content on the site. Returns the URL, route file path, section title, and a content snippet.",
+      inputSchema: {
+        query: z.string().min(1),
+        scope: z.string().optional()
+      }
+    },
+    async (input) => {
+      const result = await engine.search({
+        q: input.query,
+        topK: 1,
+        scope: input.scope
+      });
+      if (result.results.length === 0) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify({
+                error: "No matching content found for the given query."
+              })
+            }
+          ]
+        };
+      }
+      const match = result.results[0];
+      const { url, routeFile, sectionTitle, snippet } = match;
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify({ url, routeFile, sectionTitle, snippet })
+          }
+        ]
+      };
+    }
+  );
+  server.registerTool(
+    "get_related_pages",
+    {
+      description: "Find pages related to a given URL using link graph, semantic similarity, and structural proximity. Returns related pages ranked by a composite relatedness score. Use this to discover content connected to a known page.",
+      inputSchema: {
+        pathOrUrl: z.string().min(1),
+        scope: z.string().optional(),
+        topK: z.number().int().positive().max(25).optional()
+      }
+    },
+    async (input) => {
+      const result = await engine.getRelatedPages(input.pathOrUrl, {
+        topK: input.topK,
+        scope: input.scope
+      });
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify(result, null, 2)
+          }
+        ]
+      };
+    }
+  );
   return server;
 }
+function resolveApiKey(config) {
+  return config.mcp.http.apiKey ?? (config.mcp.http.apiKeyEnv ? process.env[config.mcp.http.apiKeyEnv] : void 0);
+}
+function verifyApiKey(provided, expected) {
+  const a = createHash("sha256").update(provided).digest();
+  const b = createHash("sha256").update(expected).digest();
+  return timingSafeEqual(a, b);
+}
 function redirectConsoleToStderr() {
   console.log = (...args) => {
     process.stderr.write(`[LOG] ${args.map(String).join(" ")}
@@ -20251,7 +21900,22 @@ async function startHttpServer(serverFactory, config, opts) {
   const app = createMcpExpressApp();
   const port = opts.httpPort ?? config.mcp.http.port;
   const endpointPath = opts.httpPath ?? config.mcp.http.path;
+  const isPublic = config.mcp.access === "public";
+  const host = isPublic ? "0.0.0.0" : "127.0.0.1";
+  const apiKey = isPublic ? resolveApiKey(config) : void 0;
   app.post(endpointPath, async (req, res) => {
+    if (isPublic && apiKey) {
+      const authHeader = req.headers["authorization"];
+      const provided = (authHeader?.startsWith("Bearer ") ? authHeader.slice(7) : void 0) ?? req.headers["x-api-key"] ?? "";
+      if (!provided || !verifyApiKey(provided, apiKey)) {
+        res.status(401).json({
+          jsonrpc: "2.0",
+          error: { code: -32001, message: "Unauthorized" },
+          id: null
+        });
+        return;
+      }
+    }
     const server = serverFactory();
     const transport = new StreamableHTTPServerTransport({
       sessionIdGenerator: void 0
@@ -20301,9 +21965,12 @@ async function startHttpServer(serverFactory, config, opts) {
     );
   });
   await new Promise((resolve, reject) => {
-    const instance = app.listen(port, "127.0.0.1", () => {
-      process.stderr.write(`SearchSocket MCP HTTP server listening on http://127.0.0.1:${port}${endpointPath}
+    const instance = app.listen(port, host, () => {
+      process.stderr.write(`SearchSocket MCP HTTP server listening on http://${host}:${port}${endpointPath}
 `);
+      if (isPublic) {
+        process.stderr.write("WARNING: Server is in public mode. Ensure HTTPS is configured via a reverse proxy for production use.\n");
+      }
       resolve();
     });
     instance.once("error", reject);
@@ -20318,6 +21985,13 @@ async function runMcpServer(options = {}) {
     cwd: options.cwd,
     configPath: options.configPath
   });
+  if (options.access) config.mcp.access = options.access;
+  if (options.apiKey) config.mcp.http.apiKey = options.apiKey;
+  if (config.mcp.access === "public" && !resolveApiKey(config)) {
+    throw new Error(
+      'MCP access is "public" but no API key is configured. Pass --api-key or set mcp.http.apiKey / mcp.http.apiKeyEnv in config.'
+    );
+  }
   const resolvedTransport = options.transport ?? config.mcp.transport;
   if (resolvedTransport === "stdio") {
     redirectConsoleToStderr();
@@ -20335,8 +22009,6 @@ async function runMcpServer(options = {}) {
   const stdioTransport = new StdioServerTransport();
   await server.connect(stdioTransport);
 }
-
-// src/sveltekit/handle.ts
 var InMemoryRateLimiter = class {
   constructor(windowMs, max) {
     this.windowMs = windowMs;
@@ -20364,7 +22036,13 @@ function searchsocketHandle(options = {}) {
   let enginePromise = null;
   let configPromise = null;
   let apiPath = options.path;
+  let llmsServePath = null;
+  let serveMarkdownVariants = false;
+  let mcpPath;
+  let mcpApiKey;
+  let mcpEnableJsonResponse = true;
   let rateLimiter = null;
+  let notConfigured = false;
   const getConfig = async () => {
     if (!configPromise) {
       let configP;
@@ -20381,6 +22059,13 @@ function searchsocketHandle(options = {}) {
       }
       configPromise = configP.then((config) => {
         apiPath = apiPath ?? config.api.path;
+        mcpPath = config.mcp.handle.path;
+        mcpApiKey = config.mcp.handle.apiKey;
+        mcpEnableJsonResponse = config.mcp.handle.enableJsonResponse;
+        if (config.llmsTxt.enable) {
+          llmsServePath = "/" + config.llmsTxt.outputPath.replace(/^static\//, "");
+          serveMarkdownVariants = config.llmsTxt.serveMarkdownVariants;
+        }
         if (config.api.rateLimit && !isServerless()) {
           rateLimiter = new InMemoryRateLimiter(config.api.rateLimit.windowMs, config.api.rateLimit.max);
         }
@@ -20390,59 +22075,109 @@ function searchsocketHandle(options = {}) {
     return configPromise;
   };
   const getEngine = async () => {
+    if (notConfigured) {
+      throw new SearchSocketError(
+        "SEARCH_NOT_CONFIGURED",
+        "Search is not configured. Set the required Upstash environment variables to enable search.",
+        503
+      );
+    }
     if (!enginePromise) {
       const config = await getConfig();
       enginePromise = SearchEngine.create({
         cwd: options.cwd,
         config
+      }).catch((error) => {
+        enginePromise = null;
+        if (error instanceof SearchSocketError && error.code === "VECTOR_BACKEND_UNAVAILABLE") {
+          notConfigured = true;
+          throw new SearchSocketError(
+            "SEARCH_NOT_CONFIGURED",
+            "Search is not configured. Set the required Upstash environment variables to enable search.",
+            503
+          );
+        }
+        throw error;
       });
     }
     return enginePromise;
   };
   const bodyLimit = options.maxBodyBytes ?? 64 * 1024;
   return async ({ event, resolve }) => {
-    if (apiPath && event.url.pathname !== apiPath) {
-      return resolve(event);
+    if (apiPath && !isApiPath(event.url.pathname, apiPath) && event.url.pathname !== llmsServePath) {
+      const isMarkdownVariant = event.request.method === "GET" && event.url.pathname.endsWith(".md");
+      if (mcpPath && event.url.pathname === mcpPath) {
+        return handleMcpRequest(event, mcpApiKey, mcpEnableJsonResponse, getEngine);
+      }
+      if (mcpPath) {
+        if (serveMarkdownVariants && isMarkdownVariant) ; else {
+          return resolve(event);
+        }
+      } else {
+        if (configPromise || options.config || options.rawConfig) {
+          await getConfig();
+          if (mcpPath && event.url.pathname === mcpPath) {
+            return handleMcpRequest(event, mcpApiKey, mcpEnableJsonResponse, getEngine);
+          }
+          if (!(serveMarkdownVariants && isMarkdownVariant)) {
+            return resolve(event);
+          }
+        } else {
+          return resolve(event);
+        }
+      }
     }
     const config = await getConfig();
+    if (llmsServePath && event.request.method === "GET" && event.url.pathname === llmsServePath) {
+      const cwd = options.cwd ?? process.cwd();
+      const filePath = path.resolve(cwd, config.llmsTxt.outputPath);
+      try {
+        const content = await fs8.readFile(filePath, "utf8");
+        return new Response(content, {
+          status: 200,
+          headers: { "content-type": "text/plain; charset=utf-8" }
+        });
+      } catch {
+        return resolve(event);
+      }
+    }
+    if (serveMarkdownVariants && event.request.method === "GET" && event.url.pathname.endsWith(".md")) {
+      let rawPath;
+      try {
+        rawPath = decodeURIComponent(event.url.pathname.slice(0, -3));
+      } catch {
+        return resolve(event);
+      }
+      const scope = event.url.searchParams?.get("scope") ?? void 0;
+      try {
+        const engine = await getEngine();
+        const page = await engine.getPage(rawPath, scope);
+        return new Response(page.markdown, {
+          status: 200,
+          headers: { "content-type": "text/markdown; charset=utf-8" }
+        });
+      } catch (error) {
+        if (error instanceof SearchSocketError && error.status === 404) {
+          return resolve(event);
+        }
+        throw error;
+      }
+    }
+    if (mcpPath && event.url.pathname === mcpPath) {
+      return handleMcpRequest(event, mcpApiKey, mcpEnableJsonResponse, getEngine);
+    }
     const targetPath = apiPath ?? config.api.path;
-    if (event.url.pathname !== targetPath) {
+    if (!isApiPath(event.url.pathname, targetPath)) {
       return resolve(event);
     }
-    if (event.request.method === "OPTIONS") {
+    const subPath = event.url.pathname.slice(targetPath.length);
+    const method = event.request.method;
+    if (method === "OPTIONS") {
       return new Response(null, {
         status: 204,
         headers: buildCorsHeaders(event.request, config)
       });
     }
-    if (event.request.method !== "POST") {
-      return withCors(
-        new Response(JSON.stringify(toErrorPayload(new SearchSocketError("INVALID_REQUEST", "Method not allowed", 405))), {
-          status: 405,
-          headers: {
-            "content-type": "application/json"
-          }
-        }),
-        event.request,
-        config
-      );
-    }
-    const contentLength = Number(event.request.headers.get("content-length") ?? 0);
-    if (contentLength > bodyLimit) {
-      return withCors(
-        new Response(
-          JSON.stringify(toErrorPayload(new SearchSocketError("INVALID_REQUEST", "Request body too large", 413))),
-          {
-            status: 413,
-            headers: {
-              "content-type": "application/json"
-            }
-          }
-        ),
-        event.request,
-        config
-      );
-    }
     if (rateLimiter) {
       const ip = event.getClientAddress?.() ?? event.request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? "unknown";
       if (!rateLimiter.check(ip)) {
@@ -20462,39 +22197,32 @@ function searchsocketHandle(options = {}) {
       }
     }
     try {
-      let rawBody;
-      if (typeof event.request.text === "function") {
-        rawBody = await event.request.text();
-      } else {
-        let parsedFallback;
-        try {
-          parsedFallback = await event.request.json();
-        } catch (error) {
-          if (error instanceof SyntaxError) {
-            throw new SearchSocketError("INVALID_REQUEST", "Malformed JSON request body", 400);
-          }
-          throw error;
+      if (method === "GET") {
+        if (subPath === "" || subPath === "/") {
+          return await handleGetSearch(event, config, getEngine);
         }
-        rawBody = JSON.stringify(parsedFallback);
+        if (subPath === "/health") {
+          return await handleGetHealth(event, config, getEngine);
+        }
+        if (subPath.startsWith("/pages/")) {
+          return await handleGetPage(event, config, getEngine, subPath);
+        }
+        return withCors(
+          new Response(JSON.stringify(toErrorPayload(new SearchSocketError("INVALID_REQUEST", "Not found", 404))), {
+            status: 404,
+            headers: { "content-type": "application/json" }
+          }),
+          event.request,
+          config
+        );
       }
-      if (Buffer.byteLength(rawBody, "utf8") > bodyLimit) {
-        throw new SearchSocketError("INVALID_REQUEST", "Request body too large", 413);
+      if (method === "POST" && (subPath === "" || subPath === "/")) {
+        return await handlePostSearch(event, config, getEngine, bodyLimit);
       }
-      let body;
-      try {
-        body = JSON.parse(rawBody);
-      } catch {
-        throw new SearchSocketError("INVALID_REQUEST", "Malformed JSON request body", 400);
-      }
-      const engine = await getEngine();
-      const searchRequest = body;
-      const result = await engine.search(searchRequest);
       return withCors(
-        new Response(JSON.stringify(result), {
-          status: 200,
-          headers: {
-            "content-type": "application/json"
-          }
+        new Response(JSON.stringify(toErrorPayload(new SearchSocketError("INVALID_REQUEST", "Method not allowed", 405))), {
+          status: 405,
+          headers: { "content-type": "application/json" }
         }),
         event.request,
         config
@@ -20515,6 +22243,183 @@ function searchsocketHandle(options = {}) {
     }
   };
 }
+function isApiPath(pathname, apiPath) {
+  return pathname === apiPath || pathname.startsWith(apiPath + "/");
+}
+async function handleGetSearch(event, config, getEngine) {
+  const params = event.url.searchParams;
+  const q = params.get("q");
+  if (!q || q.trim() === "") {
+    throw new SearchSocketError("INVALID_REQUEST", "Missing required query parameter: q", 400);
+  }
+  const searchRequest = { q };
+  const topK = params.get("topK");
+  if (topK !== null) {
+    const parsed = Number.parseInt(topK, 10);
+    if (Number.isNaN(parsed) || parsed < 1) {
+      throw new SearchSocketError("INVALID_REQUEST", "topK must be a positive integer", 400);
+    }
+    searchRequest.topK = parsed;
+  }
+  const scope = params.get("scope");
+  if (scope !== null) searchRequest.scope = scope;
+  const pathPrefix = params.get("pathPrefix");
+  if (pathPrefix !== null) searchRequest.pathPrefix = pathPrefix;
+  const groupBy = params.get("groupBy");
+  if (groupBy) {
+    if (groupBy !== "page" && groupBy !== "chunk") {
+      throw new SearchSocketError("INVALID_REQUEST", 'groupBy must be "page" or "chunk"', 400);
+    }
+    searchRequest.groupBy = groupBy;
+  }
+  const maxSubResults = params.get("maxSubResults");
+  if (maxSubResults !== null) {
+    const parsed = Number.parseInt(maxSubResults, 10);
+    if (Number.isNaN(parsed) || parsed < 1 || parsed > 20) {
+      throw new SearchSocketError("INVALID_REQUEST", "maxSubResults must be a positive integer between 1 and 20", 400);
+    }
+    searchRequest.maxSubResults = parsed;
+  }
+  const tags = params.getAll("tags");
+  if (tags.length > 0) searchRequest.tags = tags;
+  const engine = await getEngine();
+  const result = await engine.search(searchRequest);
+  return withCors(
+    new Response(JSON.stringify(result), {
+      status: 200,
+      headers: { "content-type": "application/json" }
+    }),
+    event.request,
+    config
+  );
+}
+async function handleGetHealth(event, config, getEngine) {
+  const engine = await getEngine();
+  const result = await engine.health();
+  return withCors(
+    new Response(JSON.stringify(result), {
+      status: 200,
+      headers: { "content-type": "application/json" }
+    }),
+    event.request,
+    config
+  );
+}
+async function handleGetPage(event, config, getEngine, subPath) {
+  const rawPath = subPath.slice("/pages".length);
+  let pagePath;
+  try {
+    pagePath = decodeURIComponent(rawPath);
+  } catch {
+    throw new SearchSocketError("INVALID_REQUEST", "Malformed page path", 400);
+  }
+  const scope = event.url.searchParams?.get("scope") ?? void 0;
+  const engine = await getEngine();
+  const result = await engine.getPage(pagePath, scope);
+  return withCors(
+    new Response(JSON.stringify(result), {
+      status: 200,
+      headers: { "content-type": "application/json" }
+    }),
+    event.request,
+    config
+  );
+}
+async function handlePostSearch(event, config, getEngine, bodyLimit) {
+  const contentLength = Number(event.request.headers.get("content-length") ?? 0);
+  if (contentLength > bodyLimit) {
+    throw new SearchSocketError("INVALID_REQUEST", "Request body too large", 413);
+  }
+  let rawBody;
+  if (typeof event.request.text === "function") {
+    rawBody = await event.request.text();
+  } else {
+    let parsedFallback;
+    try {
+      parsedFallback = await event.request.json();
+    } catch (error) {
+      if (error instanceof SyntaxError) {
+        throw new SearchSocketError("INVALID_REQUEST", "Malformed JSON request body", 400);
+      }
+      throw error;
+    }
+    rawBody = JSON.stringify(parsedFallback);
+  }
+  if (Buffer.byteLength(rawBody, "utf8") > bodyLimit) {
+    throw new SearchSocketError("INVALID_REQUEST", "Request body too large", 413);
+  }
+  let body;
+  try {
+    body = JSON.parse(rawBody);
+  } catch {
+    throw new SearchSocketError("INVALID_REQUEST", "Malformed JSON request body", 400);
+  }
+  const engine = await getEngine();
+  const searchRequest = body;
+  const result = await engine.search(searchRequest);
+  return withCors(
+    new Response(JSON.stringify(result), {
+      status: 200,
+      headers: { "content-type": "application/json" }
+    }),
+    event.request,
+    config
+  );
+}
+async function handleMcpRequest(event, apiKey, enableJsonResponse, getEngine) {
+  if (apiKey) {
+    const authHeader = event.request.headers.get("authorization") ?? "";
+    const token = authHeader.startsWith("Bearer ") ? authHeader.slice(7) : "";
+    const tokenBuf = Buffer.from(token);
+    const keyBuf = Buffer.from(apiKey);
+    if (tokenBuf.length !== keyBuf.length || !timingSafeEqual(tokenBuf, keyBuf)) {
+      return new Response(
+        JSON.stringify({
+          jsonrpc: "2.0",
+          error: { code: -32001, message: "Unauthorized" },
+          id: null
+        }),
+        { status: 401, headers: { "content-type": "application/json" } }
+      );
+    }
+  }
+  const transport = new WebStandardStreamableHTTPServerTransport({
+    sessionIdGenerator: void 0,
+    enableJsonResponse
+  });
+  let server;
+  try {
+    const engine = await getEngine();
+    server = createServer(engine);
+    await server.connect(transport);
+    const response = await transport.handleRequest(event.request);
+    if (enableJsonResponse) {
+      await transport.close();
+      await server.close();
+    }
+    return response;
+  } catch (error) {
+    try {
+      await transport.close();
+    } catch {
+    }
+    try {
+      await server?.close();
+    } catch {
+    }
+    return new Response(
+      JSON.stringify({
+        jsonrpc: "2.0",
+        error: {
+          code: -32603,
+          message: error instanceof Error ? error.message : "Internal server error"
+        },
+        id: null
+      }),
+      { status: 500, headers: { "content-type": "application/json" } }
+    );
+  }
+}
 function buildCorsHeaders(request, config) {
   const allowOrigins = config.api.cors.allowOrigins;
   if (!allowOrigins || allowOrigins.length === 0) {
@@ -20527,7 +22432,7 @@ function buildCorsHeaders(request, config) {
   }
   return {
     "access-control-allow-origin": allowOrigins.includes("*") ? "*" : origin,
-    "access-control-allow-methods": "POST, OPTIONS",
+    "access-control-allow-methods": "GET, POST, OPTIONS",
     "access-control-allow-headers": "content-type"
   };
 }
@@ -20563,9 +22468,6 @@ function shouldRunAutoIndex(options) {
   if (explicit && /^(1|true|yes)$/i.test(explicit)) {
     return true;
   }
-  if (process.env.CI && /^(1|true)$/i.test(process.env.CI)) {
-    return true;
-  }
   return false;
 }
 function searchsocketVitePlugin(options = {}) {
@@ -20590,7 +22492,8 @@ function searchsocketVitePlugin(options = {}) {
         const pipeline = await IndexPipeline.create({
           cwd,
           configPath: options.configPath,
-          logger: logger3
+          logger: logger3,
+          hooks: options.hooks
         });
         const stats = await pipeline.run({
           changedOnly: options.changedOnly ?? true,