searchsocket 0.2.1 → 0.3.1

package/dist/cli.js

@@ -12,7 +12,7 @@ import { Command } from "commander";
 // package.json
 var package_default = {
   name: "searchsocket",
-  version: "0.2.1",
+  version: "0.3.1",
   description: "Semantic site search and MCP retrieval for SvelteKit static sites",
   license: "MIT",
   author: "Greg Priday <greg@siteorigin.com>",
@@ -82,7 +82,6 @@ var package_default = {
     "fast-glob": "^3.3.3",
     "gray-matter": "^4.0.3",
     jiti: "^2.6.1",
-    openai: "^6.19.0",
     "p-limit": "^7.3.0",
     turndown: "^7.2.2",
     "turndown-plugin-gfm": "^1.0.2",
@@ -133,7 +132,11 @@ var searchSocketConfigSchema = z.object({
       outputDir: z.string().min(1).optional(),
       paramValues: z.record(z.string(), z.array(z.string())).optional(),
       exclude: z.array(z.string()).optional(),
-      previewTimeout: z.number().int().positive().optional()
+      previewTimeout: z.number().int().positive().optional(),
+      discover: z.boolean().optional(),
+      seedUrls: z.array(z.string()).optional(),
+      maxPages: z.number().int().positive().optional(),
+      maxDepth: z.number().int().nonnegative().optional()
     }).optional()
   }).optional(),
   extract: z.object({
@@ -160,8 +163,9 @@ var searchSocketConfigSchema = z.object({
     pageSummaryChunk: z.boolean().optional()
   }).optional(),
   embeddings: z.object({
-    provider: z.literal("openai").optional(),
+    provider: z.literal("jina").optional(),
     model: z.string().min(1).optional(),
+    apiKey: z.string().min(1).optional(),
     apiKeyEnv: z.string().min(1).optional(),
     batchSize: z.number().int().positive().optional(),
     concurrency: z.number().int().positive().optional(),
@@ -170,18 +174,17 @@ var searchSocketConfigSchema = z.object({
   vector: z.object({
     dimension: z.number().int().positive().optional(),
     turso: z.object({
+      url: z.string().url().optional(),
+      authToken: z.string().min(1).optional(),
       urlEnv: z.string().optional(),
       authTokenEnv: z.string().optional(),
       localPath: z.string().optional()
     }).optional()
   }).optional(),
   rerank: z.object({
-    provider: z.enum(["none", "jina"]).optional(),
+    enabled: z.boolean().optional(),
     topN: z.number().int().positive().optional(),
-    jina: z.object({
-      apiKeyEnv: z.string().optional(),
-      model: z.string().optional()
-    }).optional()
+    model: z.string().optional()
   }).optional(),
   ranking: z.object({
     enableIncomingLinkBoost: z.boolean().optional(),
@@ -190,6 +193,7 @@ var searchSocketConfigSchema = z.object({
     aggregationCap: z.number().int().positive().optional(),
     aggregationDecay: z.number().min(0).max(1).optional(),
     minChunkScoreRatio: z.number().min(0).max(1).optional(),
+    minScore: z.number().min(0).max(1).optional(),
     weights: z.object({
       incomingLinks: z.number().optional(),
       depth: z.number().optional(),
@@ -270,9 +274,9 @@ function createDefaultConfig(projectId) {
       pageSummaryChunk: true
     },
     embeddings: {
-      provider: "openai",
-      model: "text-embedding-3-small",
-      apiKeyEnv: "OPENAI_API_KEY",
+      provider: "jina",
+      model: "jina-embeddings-v3",
+      apiKeyEnv: "JINA_API_KEY",
       batchSize: 64,
       concurrency: 4
     },
@@ -284,12 +288,9 @@ function createDefaultConfig(projectId) {
       }
     },
     rerank: {
-      provider: "none",
+      enabled: false,
       topN: 20,
-      jina: {
-        apiKeyEnv: "JINA_API_KEY",
-        model: "jina-reranker-v2-base-multilingual"
-      }
+      model: "jina-reranker-v2-base-multilingual"
     },
     ranking: {
       enableIncomingLinkBoost: true,
@@ -298,6 +299,7 @@ function createDefaultConfig(projectId) {
       aggregationCap: 5,
       aggregationDecay: 0.5,
       minChunkScoreRatio: 0.5,
+      minScore: 0,
       weights: {
         incomingLinks: 0.05,
         depth: 0.03,
@@ -408,7 +410,11 @@ ${issues}`
         outputDir: parsed.source.build.outputDir ?? ".svelte-kit/output",
         paramValues: parsed.source.build.paramValues ?? {},
         exclude: parsed.source.build.exclude ?? [],
-        previewTimeout: parsed.source.build.previewTimeout ?? 3e4
+        previewTimeout: parsed.source.build.previewTimeout ?? 3e4,
+        discover: parsed.source.build.discover ?? false,
+        seedUrls: parsed.source.build.seedUrls ?? ["/"],
+        maxPages: parsed.source.build.maxPages ?? 200,
+        maxDepth: parsed.source.build.maxDepth ?? 10
       } : void 0
     },
     extract: {
@@ -437,11 +443,7 @@ ${issues}`
     },
     rerank: {
       ...defaults.rerank,
-      ...parsed.rerank,
-      jina: {
-        ...defaults.rerank.jina,
-        ...parsed.rerank?.jina
-      }
+      ...parsed.rerank
     },
     ranking: {
       ...defaults.ranking,
@@ -488,7 +490,11 @@ ${issues}`
       outputDir: ".svelte-kit/output",
       paramValues: {},
       exclude: [],
-      previewTimeout: 3e4
+      previewTimeout: 3e4,
+      discover: false,
+      seedUrls: ["/"],
+      maxPages: 200,
+      maxDepth: 10
     };
   }
   if (merged.source.mode === "crawl" && !merged.source.crawl?.baseUrl) {
@@ -529,7 +535,7 @@ function writeMinimalConfig(cwd) {
     return target;
   }
   const content = `export default {
-  embeddings: { apiKeyEnv: "OPENAI_API_KEY" }
+  embeddings: { apiKeyEnv: "JINA_API_KEY" }
 };
 `;
   fs.writeFileSync(target, content, "utf8");
@@ -540,14 +546,16 @@ function writeMinimalConfig(cwd) {
 var Logger = class {
   json;
   verbose;
+  quiet;
   stderrOnly;
   constructor(opts = {}) {
     this.json = opts.json ?? false;
     this.verbose = opts.verbose ?? false;
+    this.quiet = opts.quiet ?? false;
     this.stderrOnly = opts.stderrOnly ?? false;
   }
   info(message) {
-    if (this.json) {
+    if (this.quiet || this.json) {
       return;
     }
     this.writeOut(`${message}
@@ -561,7 +569,7 @@ var Logger = class {
       this.logJson("debug", { message });
       return;
     }
-    this.writeOut(`${message}
+    this.writeOut(`  ${message}
 `);
   }
   warn(message) {
@@ -588,7 +596,7 @@ var Logger = class {
       this.logJson(event, data);
       return;
     }
-    this.writeOut(`[${event}] ${data ? JSON.stringify(data) : ""}
+    this.writeOut(`  [${event}] ${data ? JSON.stringify(data) : ""}
 `);
   }
   writeOut(text) {
@@ -695,18 +703,18 @@ function ensureStateDirs(cwd, stateDir, scope) {
   return { statePath, pagesPath };
 }
 
-// src/embeddings/openai.ts
-import OpenAI from "openai";
+// src/embeddings/jina.ts
 import pLimit from "p-limit";
 function sleep(ms) {
   return new Promise((resolve) => {
     setTimeout(resolve, ms);
   });
 }
-var OpenAIEmbeddingsProvider = class {
-  client;
+var JinaEmbeddingsProvider = class {
+  apiKey;
   batchSize;
   concurrency;
+  defaultTask;
   constructor(options) {
     if (!Number.isInteger(options.batchSize) || options.batchSize <= 0) {
       throw new Error(`Invalid batchSize: ${options.batchSize}. batchSize must be a positive integer.`);
@@ -714,11 +722,10 @@ var OpenAIEmbeddingsProvider = class {
     if (!Number.isInteger(options.concurrency) || options.concurrency <= 0) {
       throw new Error(`Invalid concurrency: ${options.concurrency}. concurrency must be a positive integer.`);
     }
-    this.client = new OpenAI({
-      apiKey: options.apiKey
-    });
+    this.apiKey = options.apiKey;
     this.batchSize = options.batchSize;
     this.concurrency = options.concurrency;
+    this.defaultTask = options.task ?? "retrieval.passage";
   }
   estimateTokens(text) {
     const normalized = text.trim();
@@ -732,7 +739,7 @@ var OpenAIEmbeddingsProvider = class {
     const lexicalEstimate = Math.ceil(wordCount * 1.25 + punctuationCount * 0.45 + cjkCount * 1.6);
     return Math.max(1, Math.max(charEstimate, lexicalEstimate));
   }
-  async embedTexts(texts, modelId) {
+  async embedTexts(texts, modelId, task) {
     if (texts.length === 0) {
       return [];
     }
@@ -748,33 +755,52 @@ var OpenAIEmbeddingsProvider = class {
     await Promise.all(
       batches.map(
         (batch, position) => limit(async () => {
-          outputs[position] = await this.embedWithRetry(batch.values, modelId);
+          outputs[position] = await this.embedWithRetry(batch.values, modelId, task ?? this.defaultTask);
         })
       )
     );
     return outputs.flat();
   }
-  async embedWithRetry(texts, modelId) {
+  async embedWithRetry(texts, modelId, task) {
     const maxAttempts = 5;
     let attempt = 0;
     while (attempt < maxAttempts) {
       attempt += 1;
+      let response;
       try {
-        const response = await this.client.embeddings.create({
-          model: modelId,
-          input: texts,
-          encoding_format: "float"
+        response = await fetch("https://api.jina.ai/v1/embeddings", {
+          method: "POST",
+          headers: {
+            "content-type": "application/json",
+            authorization: `Bearer ${this.apiKey}`
+          },
+          body: JSON.stringify({
+            model: modelId,
+            input: texts,
+            task
+          })
         });
-        return response.data.map((entry) => entry.embedding);
       } catch (error) {
-        const status = error.status;
-        const retryable = status === 429 || typeof status === "number" && status >= 500;
-        if (!retryable || attempt >= maxAttempts) {
+        if (attempt >= maxAttempts) {
           throw error;
         }
-        const delay = Math.min(2 ** attempt * 300, 5e3);
-        await sleep(delay);
+        await sleep(Math.min(2 ** attempt * 300, 5e3));
+        continue;
+      }
+      if (!response.ok) {
+        const retryable = response.status === 429 || response.status >= 500;
+        if (!retryable || attempt >= maxAttempts) {
+          const errorBody = await response.text();
+          throw new Error(`Jina embeddings failed (${response.status}): ${errorBody}`);
+        }
+        await sleep(Math.min(2 ** attempt * 300, 5e3));
+        continue;
+      }
+      const payload = await response.json();
+      if (!payload.data || !Array.isArray(payload.data)) {
+        throw new Error("Invalid Jina embeddings response format");
       }
+      return payload.data.map((entry) => entry.embedding);
     }
     throw new Error("Unreachable retry state");
   }
@@ -782,20 +808,20 @@ var OpenAIEmbeddingsProvider = class {
 
 // src/embeddings/factory.ts
 function createEmbeddingsProvider(config) {
-  if (config.embeddings.provider !== "openai") {
+  if (config.embeddings.provider !== "jina") {
     throw new SearchSocketError(
       "CONFIG_MISSING",
       `Unsupported embeddings provider ${config.embeddings.provider}`
     );
   }
-  const apiKey = process.env[config.embeddings.apiKeyEnv];
+  const apiKey = config.embeddings.apiKey ?? process.env[config.embeddings.apiKeyEnv];
   if (!apiKey) {
     throw new SearchSocketError(
       "CONFIG_MISSING",
-      `Missing embeddings API key env var: ${config.embeddings.apiKeyEnv}`
+      `Missing embeddings API key: provide embeddings.apiKey or set env var ${config.embeddings.apiKeyEnv}`
     );
   }
-  return new OpenAIEmbeddingsProvider({
+  return new JinaEmbeddingsProvider({
     apiKey,
     batchSize: config.embeddings.batchSize,
     concurrency: config.embeddings.concurrency
@@ -809,6 +835,11 @@ import path11 from "path";
 import fs3 from "fs";
 import path3 from "path";
 
+// src/core/serverless.ts
+function isServerless() {
+  return !!(process.env.VERCEL || process.env.NETLIFY || process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.FUNCTIONS_WORKER || process.env.CF_PAGES);
+}
+
 // src/vector/turso.ts
 var TursoVectorStore = class {
   client;
@@ -853,6 +884,16 @@ var TursoVectorStore = class {
   }
   async ensureChunks(dim) {
     if (this.chunksReady) return;
+    const exists = await this.chunksTableExists();
+    if (exists) {
+      const currentDim = await this.getChunksDimension();
+      if (currentDim !== null && currentDim !== dim) {
+        await this.client.batch([
+          "DROP INDEX IF EXISTS idx",
+          "DROP TABLE IF EXISTS chunks"
+        ]);
+      }
+    }
     await this.client.batch([
       `CREATE TABLE IF NOT EXISTS chunks (
         id             TEXT PRIMARY KEY,
@@ -864,12 +905,16 @@ var TursoVectorStore = class {
         section_title  TEXT NOT NULL DEFAULT '',
         heading_path   TEXT NOT NULL DEFAULT '[]',
         snippet        TEXT NOT NULL DEFAULT '',
+        chunk_text     TEXT NOT NULL DEFAULT '',
+        ordinal        INTEGER NOT NULL DEFAULT 0,
         content_hash   TEXT NOT NULL DEFAULT '',
         model_id       TEXT NOT NULL DEFAULT '',
         depth          INTEGER NOT NULL DEFAULT 0,
         incoming_links INTEGER NOT NULL DEFAULT 0,
         route_file     TEXT NOT NULL DEFAULT '',
         tags           TEXT NOT NULL DEFAULT '[]',
+        description    TEXT NOT NULL DEFAULT '',
+        keywords       TEXT NOT NULL DEFAULT '[]',
         embedding      F32_BLOB(${dim})
       )`,
       `CREATE INDEX IF NOT EXISTS idx ON chunks (libsql_vector_idx(embedding, 'metric=cosine'))`
@@ -908,6 +953,38 @@ var TursoVectorStore = class {
       throw error;
     }
   }
+  /**
+   * Read the current F32_BLOB dimension from the chunks table schema.
+   * Returns null if the table doesn't exist or the dimension can't be parsed.
+   */
+  async getChunksDimension() {
+    try {
+      const rs = await this.client.execute(
+        "SELECT sql FROM sqlite_master WHERE type='table' AND name='chunks'"
+      );
+      if (rs.rows.length === 0) return null;
+      const sql = rs.rows[0].sql;
+      const match = sql.match(/F32_BLOB\((\d+)\)/i);
+      return match ? parseInt(match[1], 10) : null;
+    } catch {
+      return null;
+    }
+  }
+  /**
+   * Drop all SearchSocket tables (chunks, registry, pages) and their indexes.
+   * Used by `clean --remote` for a full reset.
+   */
+  async dropAllTables() {
+    await this.client.batch([
+      "DROP INDEX IF EXISTS idx",
+      "DROP TABLE IF EXISTS chunks",
+      "DROP TABLE IF EXISTS registry",
+      "DROP TABLE IF EXISTS pages"
+    ]);
+    this.chunksReady = false;
+    this.registryReady = false;
+    this.pagesReady = false;
+  }
   async upsert(records, _scope) {
     if (records.length === 0) return;
     const dim = this.dimension ?? records[0].vector.length;
@@ -918,9 +995,9 @@ var TursoVectorStore = class {
       const stmts = batch.map((r) => ({
         sql: `INSERT OR REPLACE INTO chunks
               (id, project_id, scope_name, url, path, title, section_title,
-               heading_path, snippet, content_hash, model_id, depth,
-               incoming_links, route_file, tags, embedding)
-              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector(?))`,
+               heading_path, snippet, chunk_text, ordinal, content_hash, model_id, depth,
+               incoming_links, route_file, tags, description, keywords, embedding)
+              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector(?))`,
         args: [
           r.id,
           r.metadata.projectId,
@@ -931,12 +1008,16 @@ var TursoVectorStore = class {
           r.metadata.sectionTitle,
           JSON.stringify(r.metadata.headingPath),
           r.metadata.snippet,
+          r.metadata.chunkText,
+          r.metadata.ordinal,
           r.metadata.contentHash,
           r.metadata.modelId,
           r.metadata.depth,
           r.metadata.incomingLinks,
           r.metadata.routeFile,
           JSON.stringify(r.metadata.tags),
+          r.metadata.description ?? "",
+          JSON.stringify(r.metadata.keywords ?? []),
           JSON.stringify(r.vector)
         ]
       }));
@@ -949,8 +1030,10 @@ var TursoVectorStore = class {
     const queryJson = JSON.stringify(queryVector);
     const rs = await this.client.execute({
       sql: `SELECT c.id, c.project_id, c.scope_name, c.url, c.path, c.title,
-                   c.section_title, c.heading_path, c.snippet, c.content_hash,
+                   c.section_title, c.heading_path, c.snippet, c.chunk_text,
+                   c.ordinal, c.content_hash,
                    c.model_id, c.depth, c.incoming_links, c.route_file, c.tags,
+                   c.description, c.keywords,
                    vector_distance_cos(c.embedding, vector(?)) AS distance
             FROM vector_top_k('idx', vector(?), ?) AS v
             JOIN chunks AS c ON c.rowid = v.id`,
@@ -981,6 +1064,12 @@ var TursoVectorStore = class {
       }
       const distance = row.distance;
       const score = 1 - distance;
+      const description = row.description || void 0;
+      const keywords = (() => {
+        const raw = row.keywords || "[]";
+        const parsed = JSON.parse(raw);
+        return parsed.length > 0 ? parsed : void 0;
+      })();
       hits.push({
         id: row.id,
         score,
@@ -993,12 +1082,16 @@ var TursoVectorStore = class {
           sectionTitle: row.section_title,
           headingPath: JSON.parse(row.heading_path || "[]"),
           snippet: row.snippet,
+          chunkText: row.chunk_text || "",
+          ordinal: row.ordinal || 0,
           contentHash: row.content_hash,
           modelId: row.model_id,
           depth: row.depth,
           incomingLinks: row.incoming_links,
           routeFile: row.route_file,
-          tags
+          tags,
+          description,
+          keywords
         }
       });
     }
@@ -1188,10 +1281,10 @@ var TursoVectorStore = class {
 // src/vector/factory.ts
 async function createVectorStore(config, cwd) {
   const turso = config.vector.turso;
-  const remoteUrl = process.env[turso.urlEnv];
+  const remoteUrl = turso.url ?? process.env[turso.urlEnv];
   if (remoteUrl) {
     const { createClient: createClient2 } = await import("@libsql/client/http");
-    const authToken = process.env[turso.authTokenEnv];
+    const authToken = turso.authToken ?? process.env[turso.authTokenEnv];
     const client2 = createClient2({
       url: remoteUrl,
       authToken
@@ -1201,6 +1294,12 @@ async function createVectorStore(config, cwd) {
       dimension: config.vector.dimension
     });
   }
+  if (isServerless()) {
+    throw new SearchSocketError(
+      "VECTOR_BACKEND_UNAVAILABLE",
+      `No remote vector database URL found (checked vector.turso.url and env var "${turso.urlEnv}"). Local SQLite storage is not available in serverless environments. Set ${turso.urlEnv} or pass vector.turso.url directly.`
+    );
+  }
   const { createClient } = await import("@libsql/client");
   const localPath = path3.resolve(cwd, turso.localPath);
   fs3.mkdirSync(path3.dirname(localPath), { recursive: true });
@@ -1537,7 +1636,9 @@ function chunkMirrorPage(page, config, scope) {
       incomingLinks: page.incomingLinks,
       routeFile: page.routeFile,
       tags: page.tags,
-      contentHash: ""
+      contentHash: "",
+      description: page.description,
+      keywords: page.keywords
     };
     const embeddingText = buildEmbeddingText(summaryChunk, config.chunking.prependTitle);
     summaryChunk.contentHash = sha256(normalizeText(embeddingText));
@@ -1564,7 +1665,9 @@ function chunkMirrorPage(page, config, scope) {
       incomingLinks: page.incomingLinks,
       routeFile: page.routeFile,
       tags: page.tags,
-      contentHash: ""
+      contentHash: "",
+      description: page.description,
+      keywords: page.keywords
     };
     const embeddingText = buildEmbeddingText(chunk, config.chunking.prependTitle);
     chunk.contentHash = sha256(normalizeText(embeddingText));
@@ -1828,6 +1931,7 @@ function mapUrlToRoute(urlPath, patterns) {
 }
 
 // src/indexing/sources/build/index.ts
+import { load as cheerioLoad } from "cheerio";
 import pLimit2 from "p-limit";
 
 // src/indexing/sources/build/manifest-parser.ts
@@ -2004,11 +2108,108 @@ async function startPreviewServer(cwd, options, logger3) {
 
 // src/indexing/sources/build/index.ts
 var logger = new Logger();
+function extractLinksFromHtml(html, pageUrl, baseOrigin) {
+  const $ = cheerioLoad(html);
+  const links = [];
+  $("a[href]").each((_i, el) => {
+    const href = $(el).attr("href");
+    if (!href || href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) {
+      return;
+    }
+    try {
+      const resolved = new URL(href, `${baseOrigin}${pageUrl}`);
+      if (resolved.origin !== baseOrigin) return;
+      if (!["http:", "https:"].includes(resolved.protocol)) return;
+      links.push(normalizeUrlPath(resolved.pathname));
+    } catch {
+    }
+  });
+  return [...new Set(links)];
+}
+async function discoverPages(server, buildConfig, pipelineMaxPages) {
+  const { seedUrls, maxDepth, exclude } = buildConfig;
+  const baseOrigin = new URL(server.baseUrl).origin;
+  let effectiveMax = buildConfig.maxPages;
+  if (typeof pipelineMaxPages === "number") {
+    const floored = Math.max(0, Math.floor(pipelineMaxPages));
+    effectiveMax = Math.min(effectiveMax, floored);
+  }
+  if (effectiveMax === 0) return [];
+  const visited = /* @__PURE__ */ new Set();
+  const pages = [];
+  const queue = [];
+  const limit = pLimit2(8);
+  for (const seed of seedUrls) {
+    const normalized = normalizeUrlPath(seed);
+    if (!visited.has(normalized) && !isExcluded(normalized, exclude)) {
+      visited.add(normalized);
+      queue.push({ url: normalized, depth: 0 });
+    }
+  }
+  while (queue.length > 0 && pages.length < effectiveMax) {
+    const remaining = effectiveMax - pages.length;
+    const batch = queue.splice(0, remaining);
+    const results = await Promise.allSettled(
+      batch.map(
+        (item) => limit(async () => {
+          const fullUrl = joinUrl(server.baseUrl, item.url);
+          const response = await fetch(fullUrl);
+          if (!response.ok) {
+            logger.warn(`Skipping ${item.url}: ${response.status} ${response.statusText}`);
+            return null;
+          }
+          const contentType = response.headers.get("content-type") ?? "";
+          if (!contentType.includes("text/html")) {
+            return null;
+          }
+          const html = await response.text();
+          if (item.depth < maxDepth) {
+            const links = extractLinksFromHtml(html, item.url, baseOrigin);
+            for (const link of links) {
+              if (!visited.has(link) && !isExcluded(link, exclude)) {
+                visited.add(link);
+                queue.push({ url: link, depth: item.depth + 1 });
+              }
+            }
+          }
+          return {
+            url: item.url,
+            html,
+            sourcePath: fullUrl,
+            outgoingLinks: []
+          };
+        })
+      )
+    );
+    for (const result of results) {
+      if (result.status === "fulfilled" && result.value) {
+        pages.push(result.value);
+      }
+    }
+  }
+  if (pages.length >= effectiveMax && queue.length > 0) {
+    logger.warn(`Discovery crawl reached maxPages limit (${effectiveMax}), ${queue.length} URLs not visited.`);
+  }
+  logger.event("build_discover_complete", {
+    pagesFound: pages.length,
+    urlsVisited: visited.size,
+    urlsSkipped: queue.length
+  });
+  return pages;
+}
 async function loadBuildPages(cwd, config, maxPages) {
   const buildConfig = config.source.build;
   if (!buildConfig) {
     throw new Error("build source config is missing");
   }
+  if (buildConfig.discover) {
+    const server2 = await startPreviewServer(cwd, { previewTimeout: buildConfig.previewTimeout }, logger);
+    try {
+      return await discoverPages(server2, buildConfig, maxPages);
+    } finally {
+      await server2.shutdown();
+    }
+  }
   const routes = await parseManifest(cwd, buildConfig.outputDir);
   const expanded = expandRoutes(routes, buildConfig.paramValues, buildConfig.exclude, logger);
   logger.event("build_routes_discovered", {
@@ -2112,11 +2313,11 @@ async function loadContentFilesPages(cwd, config, maxPages) {
 
 // src/indexing/sources/crawl.ts
 import { gunzipSync } from "zlib";
-import { load as cheerioLoad } from "cheerio";
+import { load as cheerioLoad2 } from "cheerio";
 import pLimit3 from "p-limit";
 var logger2 = new Logger();
 function extractLocs(xml) {
-  const $ = cheerioLoad(xml, { xmlMode: true });
+  const $ = cheerioLoad2(xml, { xmlMode: true });
   const locs = [];
   $("loc").each((_i, el) => {
     const text = $(el).text().trim();
@@ -2127,7 +2328,7 @@ function extractLocs(xml) {
   return locs;
 }
 function isSitemapIndex(xml) {
-  const $ = cheerioLoad(xml, { xmlMode: true });
+  const $ = cheerioLoad2(xml, { xmlMode: true });
   return $("sitemapindex").length > 0;
 }
 async function fetchSitemapXml(url) {
@@ -2265,9 +2466,7 @@ function hrTimeMs(start) {
 
 // src/indexing/pipeline.ts
 var EMBEDDING_PRICE_PER_1K_TOKENS_USD = {
-  "text-embedding-3-small": 2e-5,
-  "text-embedding-3-large": 13e-5,
-  "text-embedding-ada-002": 1e-4
+  "jina-embeddings-v3": 2e-5
 };
 var DEFAULT_EMBEDDING_PRICE_PER_1K = 2e-5;
 var IndexPipeline = class _IndexPipeline {
@@ -2313,9 +2512,15 @@ var IndexPipeline = class _IndexPipeline {
     };
     const scope = resolveScope(this.config, options.scopeOverride);
     const { statePath } = ensureStateDirs(this.cwd, this.config.state.dir, scope);
+    const sourceMode = options.sourceOverride ?? this.config.source.mode;
+    this.logger.info(`Indexing scope "${scope.scopeName}" (source: ${sourceMode}, model: ${this.config.embeddings.model})`);
     if (options.force) {
+      this.logger.info("Force mode enabled \u2014 full rebuild");
       await cleanMirrorForScope(statePath, scope);
     }
+    if (options.dryRun) {
+      this.logger.info("Dry run \u2014 no writes will be performed");
+    }
     const manifestStart = stageStart();
     const existingHashes = await this.vectorStore.getContentHashes(scope);
     const existingModelId = await this.vectorStore.getScopeModelId(scope);
@@ -2326,8 +2531,9 @@ var IndexPipeline = class _IndexPipeline {
       );
     }
     stageEnd("manifest", manifestStart);
+    this.logger.debug(`Manifest: ${existingHashes.size} existing chunk hashes loaded`);
     const sourceStart = stageStart();
-    const sourceMode = options.sourceOverride ?? this.config.source.mode;
+    this.logger.info(`Loading pages (source: ${sourceMode})...`);
     let sourcePages;
     if (sourceMode === "static-output") {
       sourcePages = await loadStaticOutputPages(this.cwd, this.config, options.maxPages);
@@ -2339,10 +2545,13 @@ var IndexPipeline = class _IndexPipeline {
       sourcePages = await loadContentFilesPages(this.cwd, this.config, options.maxPages);
     }
     stageEnd("source", sourceStart);
+    this.logger.info(`Loaded ${sourcePages.length} page${sourcePages.length === 1 ? "" : "s"} (${stageTimingsMs["source"]}ms)`);
     const routeStart = stageStart();
     const routePatterns = await buildRoutePatterns(this.cwd);
     stageEnd("route_map", routeStart);
+    this.logger.debug(`Route mapping: ${routePatterns.length} pattern${routePatterns.length === 1 ? "" : "s"} discovered (${stageTimingsMs["route_map"]}ms)`);
     const extractStart = stageStart();
+    this.logger.info("Extracting content...");
     const extractedPages = [];
     for (const sourcePage of sourcePages) {
       const extracted = sourcePage.html ? extractFromHtml(sourcePage.url, sourcePage.html, this.config) : extractFromMarkdown(sourcePage.url, sourcePage.markdown ?? "", sourcePage.title);
@@ -2371,6 +2580,8 @@ var IndexPipeline = class _IndexPipeline {
       uniquePages.push(page);
     }
     stageEnd("extract", extractStart);
+    const skippedPages = sourcePages.length - uniquePages.length;
+    this.logger.info(`Extracted ${uniquePages.length} page${uniquePages.length === 1 ? "" : "s"}${skippedPages > 0 ? ` (${skippedPages} skipped)` : ""} (${stageTimingsMs["extract"]}ms)`);
     const linkStart = stageStart();
     const pageSet = new Set(uniquePages.map((page) => normalizeUrlPath(page.url)));
     const incomingLinkCount = /* @__PURE__ */ new Map();
@@ -2386,7 +2597,9 @@ var IndexPipeline = class _IndexPipeline {
       }
     }
     stageEnd("links", linkStart);
+    this.logger.debug(`Link analysis: computed incoming links for ${incomingLinkCount.size} pages (${stageTimingsMs["links"]}ms)`);
     const mirrorStart = stageStart();
+    this.logger.info("Writing mirror pages...");
     const mirrorPages = [];
     let routeExact = 0;
     let routeBestEffort = 0;
@@ -2456,7 +2669,9 @@ var IndexPipeline = class _IndexPipeline {
       await this.vectorStore.upsertPages(pageRecords, scope);
     }
     stageEnd("mirror", mirrorStart);
+    this.logger.info(`Mirrored ${mirrorPages.length} page${mirrorPages.length === 1 ? "" : "s"} (${routeExact} exact, ${routeBestEffort} best-effort) (${stageTimingsMs["mirror"]}ms)`);
     const chunkStart = stageStart();
+    this.logger.info("Chunking pages...");
     let chunks = mirrorPages.flatMap((page) => chunkMirrorPage(page, this.config, scope));
     const maxChunks = typeof options.maxChunks === "number" ? Math.max(0, Math.floor(options.maxChunks)) : void 0;
     if (typeof maxChunks === "number") {
@@ -2469,6 +2684,7 @@ var IndexPipeline = class _IndexPipeline {
       });
     }
     stageEnd("chunk", chunkStart);
+    this.logger.info(`Chunked into ${chunks.length} chunk${chunks.length === 1 ? "" : "s"} (${stageTimingsMs["chunk"]}ms)`);
     const currentChunkMap = /* @__PURE__ */ new Map();
     for (const chunk of chunks) {
       currentChunkMap.set(chunk.chunkKey, chunk);
@@ -2487,6 +2703,7 @@ var IndexPipeline = class _IndexPipeline {
       return existingHash !== chunk.contentHash;
     });
     const deletes = [...existingHashes.keys()].filter((chunkKey) => !currentChunkMap.has(chunkKey));
+    this.logger.info(`Changes detected: ${changedChunks.length} changed, ${deletes.length} deleted, ${chunks.length - changedChunks.length} unchanged`);
     const embedStart = stageStart();
     const chunkTokenEstimates = /* @__PURE__ */ new Map();
     for (const chunk of changedChunks) {
@@ -2501,9 +2718,11 @@ var IndexPipeline = class _IndexPipeline {
     let newEmbeddings = 0;
     const vectorsByChunk = /* @__PURE__ */ new Map();
     if (!options.dryRun && changedChunks.length > 0) {
+      this.logger.info(`Embedding ${changedChunks.length} chunk${changedChunks.length === 1 ? "" : "s"} (~${estimatedTokens.toLocaleString()} tokens, ~$${estimatedCostUSD.toFixed(6)})...`);
       const embeddings = await this.embeddings.embedTexts(
         changedChunks.map((chunk) => buildEmbeddingText(chunk, this.config.chunking.prependTitle)),
-        this.config.embeddings.model
+        this.config.embeddings.model,
+        "retrieval.passage"
       );
       if (embeddings.length !== changedChunks.length) {
         throw new SearchSocketError(
@@ -2526,8 +2745,14 @@ var IndexPipeline = class _IndexPipeline {
       }
     }
     stageEnd("embedding", embedStart);
+    if (changedChunks.length > 0) {
+      this.logger.info(`Embedded ${newEmbeddings} chunk${newEmbeddings === 1 ? "" : "s"} (${stageTimingsMs["embedding"]}ms)`);
+    } else {
+      this.logger.info("No chunks to embed \u2014 all up to date");
+    }
     const syncStart = stageStart();
     if (!options.dryRun) {
+      this.logger.info("Syncing vectors...");
       const upserts = [];
       for (const chunk of changedChunks) {
         const vector = vectorsByChunk.get(chunk.chunkKey);
@@ -2546,12 +2771,16 @@ var IndexPipeline = class _IndexPipeline {
             sectionTitle: chunk.sectionTitle ?? "",
             headingPath: chunk.headingPath,
             snippet: chunk.snippet,
+            chunkText: chunk.chunkText.slice(0, 4e3),
+            ordinal: chunk.ordinal,
             contentHash: chunk.contentHash,
             modelId: this.config.embeddings.model,
             depth: chunk.depth,
             incomingLinks: chunk.incomingLinks,
             routeFile: chunk.routeFile,
-            tags: chunk.tags
+            tags: chunk.tags,
+            description: chunk.description,
+            keywords: chunk.keywords
           }
         });
       }
@@ -2565,6 +2794,7 @@ var IndexPipeline = class _IndexPipeline {
       }
     }
     stageEnd("sync", syncStart);
+    this.logger.debug(`Sync complete (${stageTimingsMs["sync"]}ms)`);
     const finalizeStart = stageStart();
     if (!options.dryRun) {
       const scopeInfo = {
@@ -2584,6 +2814,7 @@ var IndexPipeline = class _IndexPipeline {
       });
     }
     stageEnd("finalize", finalizeStart);
+    this.logger.info("Done.");
     return {
       pagesProcessed: mirrorPages.length,
       chunksTotal: chunks.length,
@@ -2693,20 +2924,17 @@ var JinaReranker = class {
 
 // src/rerank/factory.ts
 function createReranker(config) {
-  if (config.rerank.provider === "none") {
+  if (!config.rerank.enabled) {
     return null;
   }
-  if (config.rerank.provider === "jina") {
-    const apiKey = process.env[config.rerank.jina.apiKeyEnv];
-    if (!apiKey) {
-      return null;
-    }
-    return new JinaReranker({
-      apiKey,
-      model: config.rerank.jina.model
-    });
+  const apiKey = config.embeddings.apiKey ?? process.env[config.embeddings.apiKeyEnv];
+  if (!apiKey) {
+    return null;
   }
-  return null;
+  return new JinaReranker({
+    apiKey,
+    model: config.rerank.model
+  });
 }
 
 // src/search/ranking.ts
@@ -2854,7 +3082,7 @@ var SearchEngine = class _SearchEngine {
     const groupByPage = (input.groupBy ?? "page") === "page";
     const candidateK = groupByPage ? Math.max(topK * 10, 50) : Math.max(50, topK);
     const embedStart = process.hrtime.bigint();
-    const queryEmbeddings = await this.embeddings.embedTexts([input.q], this.config.embeddings.model);
+    const queryEmbeddings = await this.embeddings.embedTexts([input.q], this.config.embeddings.model, "retrieval.query");
     const queryVector = queryEmbeddings[0];
     if (!queryVector || queryVector.length === 0 || queryVector.some((value) => !Number.isFinite(value))) {
       throw new SearchSocketError("VECTOR_BACKEND_UNAVAILABLE", "Unable to create query embedding.");
@@ -2882,13 +3110,17 @@ var SearchEngine = class _SearchEngine {
       usedRerank = true;
     }
     let results;
+    const minScore = this.config.ranking.minScore;
     if (groupByPage) {
-      const pages = aggregateByPage(ordered, this.config);
+      let pages = aggregateByPage(ordered, this.config);
+      if (minScore > 0) {
+        pages = pages.filter((p) => p.pageScore >= minScore);
+      }
       const minRatio = this.config.ranking.minChunkScoreRatio;
       results = pages.slice(0, topK).map((page) => {
         const bestScore = page.bestChunk.finalScore;
-        const minScore = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
-        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minScore).slice(0, 5);
+        const minScore2 = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
+        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minScore2).slice(0, 5);
         return {
           url: page.url,
           title: page.title,
@@ -2905,6 +3137,9 @@ var SearchEngine = class _SearchEngine {
         };
       });
     } else {
+      if (minScore > 0) {
+        ordered = ordered.filter((entry) => entry.finalScore >= minScore);
+      }
       results = ordered.slice(0, topK).map(({ hit, finalScore }) => ({
         url: hit.metadata.url,
         title: hit.metadata.title,
@@ -2976,43 +3211,67 @@ var SearchEngine = class _SearchEngine {
     }
   }
   async rerankHits(query, ranked, topK) {
-    if (this.config.rerank.provider !== "jina") {
+    if (!this.config.rerank.enabled) {
       throw new SearchSocketError(
         "INVALID_REQUEST",
-        "rerank=true requested but rerank.provider is not configured as 'jina'.",
+        "rerank=true requested but rerank.enabled is not set to true.",
         400
       );
     }
     if (!this.reranker) {
       throw new SearchSocketError(
         "CONFIG_MISSING",
-        `rerank=true requested but ${this.config.rerank.jina.apiKeyEnv} is not set.`,
+        `rerank=true requested but ${this.config.embeddings.apiKeyEnv} is not set.`,
         400
       );
     }
-    const candidates = ranked.map(({ hit }) => ({
-      id: hit.id,
-      text: [hit.metadata.title, hit.metadata.sectionTitle, hit.metadata.snippet].filter(Boolean).join("\n")
-    }));
+    const pageGroups = /* @__PURE__ */ new Map();
+    for (const entry of ranked) {
+      const url = entry.hit.metadata.url;
+      const group = pageGroups.get(url);
+      if (group) group.push(entry);
+      else pageGroups.set(url, [entry]);
+    }
+    const MAX_CHUNKS_PER_PAGE = 5;
+    const MIN_CHUNKS_PER_PAGE = 1;
+    const MIN_CHUNK_SCORE_RATIO = 0.5;
+    const pageCandidates = [];
+    for (const [url, chunks] of pageGroups) {
+      const byScore = [...chunks].sort((a, b) => b.finalScore - a.finalScore);
+      const bestScore = byScore[0].finalScore;
+      const scoreFloor = Number.isFinite(bestScore) ? bestScore * MIN_CHUNK_SCORE_RATIO : Number.NEGATIVE_INFINITY;
+      const selected = byScore.filter(
+        (c, i) => i < MIN_CHUNKS_PER_PAGE || c.finalScore >= scoreFloor
+      ).slice(0, MAX_CHUNKS_PER_PAGE);
+      selected.sort((a, b) => (a.hit.metadata.ordinal ?? 0) - (b.hit.metadata.ordinal ?? 0));
+      const first = selected[0].hit.metadata;
+      const parts = [first.title];
+      if (first.description) {
+        parts.push(first.description);
+      }
+      if (first.keywords && first.keywords.length > 0) {
+        parts.push(first.keywords.join(", "));
+      }
+      const body = selected.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
+      parts.push(body);
+      pageCandidates.push({ id: url, text: parts.join("\n\n") });
+    }
     const reranked = await this.reranker.rerank(
       query,
-      candidates,
+      pageCandidates,
       Math.max(topK, this.config.rerank.topN)
     );
-    const rerankScoreById = new Map(reranked.map((entry) => [entry.id, entry.score]));
+    const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {
-      const rerankScore = rerankScoreById.get(entry.hit.id);
-      const safeBaseScore = Number.isFinite(entry.finalScore) ? entry.finalScore : Number.NEGATIVE_INFINITY;
-      if (rerankScore === void 0 || !Number.isFinite(rerankScore)) {
-        return {
-          ...entry,
-          finalScore: safeBaseScore
-        };
+      const pageScore = scoreByUrl.get(entry.hit.metadata.url);
+      const base = Number.isFinite(entry.finalScore) ? entry.finalScore : Number.NEGATIVE_INFINITY;
+      if (pageScore === void 0 || !Number.isFinite(pageScore)) {
+        return { ...entry, finalScore: base };
       }
-      const combinedScore = rerankScore * this.config.ranking.weights.rerank + safeBaseScore * 1e-3;
+      const combined = pageScore * this.config.ranking.weights.rerank + base * 1e-3;
       return {
         ...entry,
-        finalScore: Number.isFinite(combinedScore) ? combinedScore : safeBaseScore
+        finalScore: Number.isFinite(combined) ? combined : base
       };
     }).sort((a, b) => {
       const delta = b.finalScore - a.finalScore;
@@ -3332,6 +3591,7 @@ function getRootOptions(command) {
 }
 async function runIndexCommand(opts) {
   const logger3 = new Logger({
+    quiet: opts.quiet,
     verbose: opts.verbose,
     json: opts.json
   });
@@ -3355,7 +3615,9 @@ async function runIndexCommand(opts) {
 `);
     return;
   }
-  printIndexSummary(stats);
+  if (!opts.quiet) {
+    printIndexSummary(stats);
+  }
 }
 var program = new Command();
 program.name("searchsocket").description("Semantic site search and MCP retrieval for SvelteKit").version(package_default.version).option("-C, --cwd <path>", "working directory", process.cwd()).option("--config <path>", "config path (defaults to searchsocket.config.ts)");
@@ -3379,7 +3641,7 @@ program.command("init").description("Create searchsocket.config.ts and .searchso
   process.stdout.write("// searchsocketVitePlugin({ enabled: true, changedOnly: true })\n");
   process.stdout.write("// or env-driven: SEARCHSOCKET_AUTO_INDEX=1 pnpm build\n");
 });
-program.command("index").description("Index site content into markdown mirror + vector store").option("--scope <name>", "scope override").option("--changed-only", "only process changed chunks", true).option("--no-changed-only", "re-index regardless of previous manifest").option("--force", "force full mirror rebuild and re-upsert", false).option("--dry-run", "compute plan and cost, no API writes", false).option("--source <mode>", "source mode override: static-output|crawl|content-files|build").option("--max-pages <n>", "limit pages processed").option("--max-chunks <n>", "limit chunks processed").option("--verbose", "verbose output", false).option("--json", "emit JSON logs and summary", false).action(async (opts, command) => {
+program.command("index").description("Index site content into markdown mirror + vector store").option("--scope <name>", "scope override").option("--changed-only", "only process changed chunks", true).option("--no-changed-only", "re-index regardless of previous manifest").option("--force", "force full mirror rebuild and re-upsert", false).option("--dry-run", "compute plan and cost, no API writes", false).option("--source <mode>", "source mode override: static-output|crawl|content-files|build").option("--max-pages <n>", "limit pages processed").option("--max-chunks <n>", "limit chunks processed").option("--quiet", "suppress all output except errors and warnings", false).option("--verbose", "verbose output", false).option("--json", "emit JSON logs and summary", false).action(async (opts, command) => {
   const rootOpts = getRootOptions(command);
   const cwd = path13.resolve(rootOpts?.cwd ?? process.cwd());
   await runIndexCommand({
@@ -3392,6 +3654,7 @@ program.command("index").description("Index site content into markdown mirror +
     source: opts.source,
     maxPages: opts.maxPages ? parsePositiveInt(opts.maxPages, "--max-pages") : void 0,
     maxChunks: opts.maxChunks ? parsePositiveInt(opts.maxChunks, "--max-chunks") : void 0,
+    quiet: opts.quiet,
     verbose: opts.verbose,
     json: opts.json
   });
@@ -3554,8 +3817,8 @@ program.command("clean").description("Delete local state and optionally delete r
 `);
   if (opts.remote) {
     const vectorStore = await createVectorStore(config, cwd);
-    await vectorStore.deleteScope(scope);
-    process.stdout.write(`deleted remote vectors for scope ${scope.scopeName}
+    await vectorStore.dropAllTables();
+    process.stdout.write(`dropped all remote tables (chunks, registry, pages)
 `);
   }
 });
@@ -3680,14 +3943,6 @@ program.command("doctor").description("Validate config, env vars, provider conne
         details: tursoUrl ? `remote: ${tursoUrl}` : `local file: ${config.vector.turso.localPath}`
       });
     }
-    if (config.rerank.provider === "jina") {
-      const jinaKey = process.env[config.rerank.jina.apiKeyEnv];
-      checks.push({
-        name: `env ${config.rerank.jina.apiKeyEnv}`,
-        ok: Boolean(jinaKey),
-        details: jinaKey ? void 0 : "missing"
-      });
-    }
     if (config.source.mode === "static-output") {
       const outputDir = path13.resolve(cwd, config.source.staticOutputDir);
       const exists = fs9.existsSync(outputDir);