searchsocket 0.2.1 → 0.3.1

package/dist/sveltekit.cjs

@@ -4,8 +4,7 @@ var fs = require('fs');
 var path = require('path');
 var jiti = require('jiti');
 var zod = require('zod');
-var OpenAI = require('openai');
-var pLimit = require('p-limit');
+var pLimit2 = require('p-limit');
 var child_process = require('child_process');
 var crypto = require('crypto');
 var cheerio = require('cheerio');
@@ -19,8 +18,7 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
 
 var fs__default = /*#__PURE__*/_interopDefault(fs);
 var path__default = /*#__PURE__*/_interopDefault(path);
-var OpenAI__default = /*#__PURE__*/_interopDefault(OpenAI);
-var pLimit__default = /*#__PURE__*/_interopDefault(pLimit);
+var pLimit2__default = /*#__PURE__*/_interopDefault(pLimit2);
 var matter__default = /*#__PURE__*/_interopDefault(matter);
 var fs4__default = /*#__PURE__*/_interopDefault(fs4);
 var fg__default = /*#__PURE__*/_interopDefault(fg);
@@ -16629,7 +16627,11 @@ var searchSocketConfigSchema = zod.z.object({
       outputDir: zod.z.string().min(1).optional(),
       paramValues: zod.z.record(zod.z.string(), zod.z.array(zod.z.string())).optional(),
       exclude: zod.z.array(zod.z.string()).optional(),
-      previewTimeout: zod.z.number().int().positive().optional()
+      previewTimeout: zod.z.number().int().positive().optional(),
+      discover: zod.z.boolean().optional(),
+      seedUrls: zod.z.array(zod.z.string()).optional(),
+      maxPages: zod.z.number().int().positive().optional(),
+      maxDepth: zod.z.number().int().nonnegative().optional()
     }).optional()
   }).optional(),
   extract: zod.z.object({
@@ -16656,8 +16658,9 @@ var searchSocketConfigSchema = zod.z.object({
     pageSummaryChunk: zod.z.boolean().optional()
   }).optional(),
   embeddings: zod.z.object({
-    provider: zod.z.literal("openai").optional(),
+    provider: zod.z.literal("jina").optional(),
     model: zod.z.string().min(1).optional(),
+    apiKey: zod.z.string().min(1).optional(),
     apiKeyEnv: zod.z.string().min(1).optional(),
     batchSize: zod.z.number().int().positive().optional(),
     concurrency: zod.z.number().int().positive().optional(),
@@ -16666,18 +16669,17 @@ var searchSocketConfigSchema = zod.z.object({
   vector: zod.z.object({
     dimension: zod.z.number().int().positive().optional(),
     turso: zod.z.object({
+      url: zod.z.string().url().optional(),
+      authToken: zod.z.string().min(1).optional(),
       urlEnv: zod.z.string().optional(),
       authTokenEnv: zod.z.string().optional(),
       localPath: zod.z.string().optional()
     }).optional()
   }).optional(),
   rerank: zod.z.object({
-    provider: zod.z.enum(["none", "jina"]).optional(),
+    enabled: zod.z.boolean().optional(),
     topN: zod.z.number().int().positive().optional(),
-    jina: zod.z.object({
-      apiKeyEnv: zod.z.string().optional(),
-      model: zod.z.string().optional()
-    }).optional()
+    model: zod.z.string().optional()
   }).optional(),
   ranking: zod.z.object({
     enableIncomingLinkBoost: zod.z.boolean().optional(),
@@ -16686,6 +16688,7 @@ var searchSocketConfigSchema = zod.z.object({
     aggregationCap: zod.z.number().int().positive().optional(),
     aggregationDecay: zod.z.number().min(0).max(1).optional(),
     minChunkScoreRatio: zod.z.number().min(0).max(1).optional(),
+    minScore: zod.z.number().min(0).max(1).optional(),
     weights: zod.z.object({
       incomingLinks: zod.z.number().optional(),
       depth: zod.z.number().optional(),
@@ -16766,9 +16769,9 @@ function createDefaultConfig(projectId) {
       pageSummaryChunk: true
     },
     embeddings: {
-      provider: "openai",
-      model: "text-embedding-3-small",
-      apiKeyEnv: "OPENAI_API_KEY",
+      provider: "jina",
+      model: "jina-embeddings-v3",
+      apiKeyEnv: "JINA_API_KEY",
       batchSize: 64,
       concurrency: 4
     },
@@ -16780,12 +16783,9 @@ function createDefaultConfig(projectId) {
       }
     },
     rerank: {
-      provider: "none",
+      enabled: false,
       topN: 20,
-      jina: {
-        apiKeyEnv: "JINA_API_KEY",
-        model: "jina-reranker-v2-base-multilingual"
-      }
+      model: "jina-reranker-v2-base-multilingual"
     },
     ranking: {
       enableIncomingLinkBoost: true,
@@ -16794,6 +16794,7 @@ function createDefaultConfig(projectId) {
       aggregationCap: 5,
       aggregationDecay: 0.5,
       minChunkScoreRatio: 0.5,
+      minScore: 0,
       weights: {
         incomingLinks: 0.05,
         depth: 0.03,
@@ -16920,7 +16921,11 @@ ${issues}`
         outputDir: parsed.source.build.outputDir ?? ".svelte-kit/output",
         paramValues: parsed.source.build.paramValues ?? {},
         exclude: parsed.source.build.exclude ?? [],
-        previewTimeout: parsed.source.build.previewTimeout ?? 3e4
+        previewTimeout: parsed.source.build.previewTimeout ?? 3e4,
+        discover: parsed.source.build.discover ?? false,
+        seedUrls: parsed.source.build.seedUrls ?? ["/"],
+        maxPages: parsed.source.build.maxPages ?? 200,
+        maxDepth: parsed.source.build.maxDepth ?? 10
       } : void 0
     },
     extract: {
@@ -16949,11 +16954,7 @@ ${issues}`
     },
     rerank: {
       ...defaults.rerank,
-      ...parsed.rerank,
-      jina: {
-        ...defaults.rerank.jina,
-        ...parsed.rerank?.jina
-      }
+      ...parsed.rerank
     },
     ranking: {
       ...defaults.ranking,
@@ -17000,7 +17001,11 @@ ${issues}`
       outputDir: ".svelte-kit/output",
       paramValues: {},
       exclude: [],
-      previewTimeout: 3e4
+      previewTimeout: 3e4,
+      discover: false,
+      seedUrls: ["/"],
+      maxPages: 200,
+      maxDepth: 10
     };
   }
   if (merged.source.mode === "crawl" && !merged.source.crawl?.baseUrl) {
@@ -17035,15 +17040,21 @@ async function loadConfig(options = {}) {
   const raw = loaded.default ?? loaded;
   return mergeConfig(cwd, raw);
 }
+
+// src/core/serverless.ts
+function isServerless() {
+  return !!(process.env.VERCEL || process.env.NETLIFY || process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.FUNCTIONS_WORKER || process.env.CF_PAGES);
+}
 function sleep(ms) {
   return new Promise((resolve) => {
     setTimeout(resolve, ms);
   });
 }
-var OpenAIEmbeddingsProvider = class {
-  client;
+var JinaEmbeddingsProvider = class {
+  apiKey;
   batchSize;
   concurrency;
+  defaultTask;
   constructor(options) {
     if (!Number.isInteger(options.batchSize) || options.batchSize <= 0) {
       throw new Error(`Invalid batchSize: ${options.batchSize}. batchSize must be a positive integer.`);
@@ -17051,11 +17062,10 @@ var OpenAIEmbeddingsProvider = class {
     if (!Number.isInteger(options.concurrency) || options.concurrency <= 0) {
       throw new Error(`Invalid concurrency: ${options.concurrency}. concurrency must be a positive integer.`);
     }
-    this.client = new OpenAI__default.default({
-      apiKey: options.apiKey
-    });
+    this.apiKey = options.apiKey;
     this.batchSize = options.batchSize;
     this.concurrency = options.concurrency;
+    this.defaultTask = options.task ?? "retrieval.passage";
   }
   estimateTokens(text) {
     const normalized = text.trim();
@@ -17069,7 +17079,7 @@ var OpenAIEmbeddingsProvider = class {
     const lexicalEstimate = Math.ceil(wordCount * 1.25 + punctuationCount * 0.45 + cjkCount * 1.6);
     return Math.max(1, Math.max(charEstimate, lexicalEstimate));
   }
-  async embedTexts(texts, modelId) {
+  async embedTexts(texts, modelId, task) {
     if (texts.length === 0) {
       return [];
     }
@@ -17081,37 +17091,56 @@ var OpenAIEmbeddingsProvider = class {
       });
     }
     const outputs = new Array(batches.length);
-    const limit = pLimit__default.default(this.concurrency);
+    const limit = pLimit2__default.default(this.concurrency);
     await Promise.all(
       batches.map(
         (batch, position) => limit(async () => {
-          outputs[position] = await this.embedWithRetry(batch.values, modelId);
+          outputs[position] = await this.embedWithRetry(batch.values, modelId, task ?? this.defaultTask);
         })
       )
     );
     return outputs.flat();
   }
-  async embedWithRetry(texts, modelId) {
+  async embedWithRetry(texts, modelId, task) {
     const maxAttempts = 5;
     let attempt = 0;
     while (attempt < maxAttempts) {
       attempt += 1;
+      let response;
       try {
-        const response = await this.client.embeddings.create({
-          model: modelId,
-          input: texts,
-          encoding_format: "float"
+        response = await fetch("https://api.jina.ai/v1/embeddings", {
+          method: "POST",
+          headers: {
+            "content-type": "application/json",
+            authorization: `Bearer ${this.apiKey}`
+          },
+          body: JSON.stringify({
+            model: modelId,
+            input: texts,
+            task
+          })
         });
-        return response.data.map((entry) => entry.embedding);
       } catch (error) {
-        const status = error.status;
-        const retryable = status === 429 || typeof status === "number" && status >= 500;
-        if (!retryable || attempt >= maxAttempts) {
+        if (attempt >= maxAttempts) {
           throw error;
         }
-        const delay = Math.min(2 ** attempt * 300, 5e3);
-        await sleep(delay);
+        await sleep(Math.min(2 ** attempt * 300, 5e3));
+        continue;
       }
+      if (!response.ok) {
+        const retryable = response.status === 429 || response.status >= 500;
+        if (!retryable || attempt >= maxAttempts) {
+          const errorBody = await response.text();
+          throw new Error(`Jina embeddings failed (${response.status}): ${errorBody}`);
+        }
+        await sleep(Math.min(2 ** attempt * 300, 5e3));
+        continue;
+      }
+      const payload = await response.json();
+      if (!payload.data || !Array.isArray(payload.data)) {
+        throw new Error("Invalid Jina embeddings response format");
+      }
+      return payload.data.map((entry) => entry.embedding);
     }
     throw new Error("Unreachable retry state");
   }
@@ -17119,20 +17148,20 @@ var OpenAIEmbeddingsProvider = class {
 
 // src/embeddings/factory.ts
 function createEmbeddingsProvider(config) {
-  if (config.embeddings.provider !== "openai") {
+  if (config.embeddings.provider !== "jina") {
     throw new SearchSocketError(
       "CONFIG_MISSING",
       `Unsupported embeddings provider ${config.embeddings.provider}`
     );
   }
-  const apiKey = process.env[config.embeddings.apiKeyEnv];
+  const apiKey = config.embeddings.apiKey ?? process.env[config.embeddings.apiKeyEnv];
   if (!apiKey) {
     throw new SearchSocketError(
       "CONFIG_MISSING",
-      `Missing embeddings API key env var: ${config.embeddings.apiKeyEnv}`
+      `Missing embeddings API key: provide embeddings.apiKey or set env var ${config.embeddings.apiKeyEnv}`
     );
   }
-  return new OpenAIEmbeddingsProvider({
+  return new JinaEmbeddingsProvider({
     apiKey,
     batchSize: config.embeddings.batchSize,
     concurrency: config.embeddings.concurrency
@@ -17295,20 +17324,17 @@ var JinaReranker = class {
 
 // src/rerank/factory.ts
 function createReranker(config) {
-  if (config.rerank.provider === "none") {
+  if (!config.rerank.enabled) {
     return null;
   }
-  if (config.rerank.provider === "jina") {
-    const apiKey = process.env[config.rerank.jina.apiKeyEnv];
-    if (!apiKey) {
-      return null;
-    }
-    return new JinaReranker({
-      apiKey,
-      model: config.rerank.jina.model
-    });
+  const apiKey = config.embeddings.apiKey ?? process.env[config.embeddings.apiKeyEnv];
+  if (!apiKey) {
+    return null;
   }
-  return null;
+  return new JinaReranker({
+    apiKey,
+    model: config.rerank.model
+  });
 }
 
 // src/utils/time.ts
@@ -17413,6 +17439,16 @@ var TursoVectorStore = class {
   }
   async ensureChunks(dim) {
     if (this.chunksReady) return;
+    const exists = await this.chunksTableExists();
+    if (exists) {
+      const currentDim = await this.getChunksDimension();
+      if (currentDim !== null && currentDim !== dim) {
+        await this.client.batch([
+          "DROP INDEX IF EXISTS idx",
+          "DROP TABLE IF EXISTS chunks"
+        ]);
+      }
+    }
     await this.client.batch([
       `CREATE TABLE IF NOT EXISTS chunks (
         id             TEXT PRIMARY KEY,
@@ -17424,12 +17460,16 @@ var TursoVectorStore = class {
         section_title  TEXT NOT NULL DEFAULT '',
         heading_path   TEXT NOT NULL DEFAULT '[]',
         snippet        TEXT NOT NULL DEFAULT '',
+        chunk_text     TEXT NOT NULL DEFAULT '',
+        ordinal        INTEGER NOT NULL DEFAULT 0,
         content_hash   TEXT NOT NULL DEFAULT '',
         model_id       TEXT NOT NULL DEFAULT '',
         depth          INTEGER NOT NULL DEFAULT 0,
         incoming_links INTEGER NOT NULL DEFAULT 0,
         route_file     TEXT NOT NULL DEFAULT '',
         tags           TEXT NOT NULL DEFAULT '[]',
+        description    TEXT NOT NULL DEFAULT '',
+        keywords       TEXT NOT NULL DEFAULT '[]',
         embedding      F32_BLOB(${dim})
       )`,
       `CREATE INDEX IF NOT EXISTS idx ON chunks (libsql_vector_idx(embedding, 'metric=cosine'))`
@@ -17468,6 +17508,38 @@ var TursoVectorStore = class {
       throw error;
     }
   }
+  /**
+   * Read the current F32_BLOB dimension from the chunks table schema.
+   * Returns null if the table doesn't exist or the dimension can't be parsed.
+   */
+  async getChunksDimension() {
+    try {
+      const rs = await this.client.execute(
+        "SELECT sql FROM sqlite_master WHERE type='table' AND name='chunks'"
+      );
+      if (rs.rows.length === 0) return null;
+      const sql = rs.rows[0].sql;
+      const match = sql.match(/F32_BLOB\((\d+)\)/i);
+      return match ? parseInt(match[1], 10) : null;
+    } catch {
+      return null;
+    }
+  }
+  /**
+   * Drop all SearchSocket tables (chunks, registry, pages) and their indexes.
+   * Used by `clean --remote` for a full reset.
+   */
+  async dropAllTables() {
+    await this.client.batch([
+      "DROP INDEX IF EXISTS idx",
+      "DROP TABLE IF EXISTS chunks",
+      "DROP TABLE IF EXISTS registry",
+      "DROP TABLE IF EXISTS pages"
+    ]);
+    this.chunksReady = false;
+    this.registryReady = false;
+    this.pagesReady = false;
+  }
   async upsert(records, _scope) {
     if (records.length === 0) return;
     const dim = this.dimension ?? records[0].vector.length;
@@ -17478,9 +17550,9 @@ var TursoVectorStore = class {
       const stmts = batch.map((r) => ({
         sql: `INSERT OR REPLACE INTO chunks
               (id, project_id, scope_name, url, path, title, section_title,
-               heading_path, snippet, content_hash, model_id, depth,
-               incoming_links, route_file, tags, embedding)
-              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector(?))`,
+               heading_path, snippet, chunk_text, ordinal, content_hash, model_id, depth,
+               incoming_links, route_file, tags, description, keywords, embedding)
+              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector(?))`,
         args: [
           r.id,
           r.metadata.projectId,
@@ -17491,12 +17563,16 @@ var TursoVectorStore = class {
           r.metadata.sectionTitle,
           JSON.stringify(r.metadata.headingPath),
           r.metadata.snippet,
+          r.metadata.chunkText,
+          r.metadata.ordinal,
           r.metadata.contentHash,
           r.metadata.modelId,
           r.metadata.depth,
           r.metadata.incomingLinks,
           r.metadata.routeFile,
           JSON.stringify(r.metadata.tags),
+          r.metadata.description ?? "",
+          JSON.stringify(r.metadata.keywords ?? []),
           JSON.stringify(r.vector)
         ]
       }));
@@ -17509,8 +17585,10 @@ var TursoVectorStore = class {
     const queryJson = JSON.stringify(queryVector);
     const rs = await this.client.execute({
       sql: `SELECT c.id, c.project_id, c.scope_name, c.url, c.path, c.title,
-                   c.section_title, c.heading_path, c.snippet, c.content_hash,
+                   c.section_title, c.heading_path, c.snippet, c.chunk_text,
+                   c.ordinal, c.content_hash,
                    c.model_id, c.depth, c.incoming_links, c.route_file, c.tags,
+                   c.description, c.keywords,
                    vector_distance_cos(c.embedding, vector(?)) AS distance
             FROM vector_top_k('idx', vector(?), ?) AS v
             JOIN chunks AS c ON c.rowid = v.id`,
@@ -17541,6 +17619,12 @@ var TursoVectorStore = class {
       }
       const distance = row.distance;
       const score = 1 - distance;
+      const description = row.description || void 0;
+      const keywords = (() => {
+        const raw = row.keywords || "[]";
+        const parsed = JSON.parse(raw);
+        return parsed.length > 0 ? parsed : void 0;
+      })();
       hits.push({
         id: row.id,
         score,
@@ -17553,12 +17637,16 @@ var TursoVectorStore = class {
           sectionTitle: row.section_title,
           headingPath: JSON.parse(row.heading_path || "[]"),
           snippet: row.snippet,
+          chunkText: row.chunk_text || "",
+          ordinal: row.ordinal || 0,
           contentHash: row.content_hash,
           modelId: row.model_id,
           depth: row.depth,
           incomingLinks: row.incoming_links,
           routeFile: row.route_file,
-          tags
+          tags,
+          description,
+          keywords
         }
       });
     }
@@ -17748,10 +17836,10 @@ var TursoVectorStore = class {
 // src/vector/factory.ts
 async function createVectorStore(config, cwd) {
   const turso = config.vector.turso;
-  const remoteUrl = process.env[turso.urlEnv];
+  const remoteUrl = turso.url ?? process.env[turso.urlEnv];
   if (remoteUrl) {
     const { createClient: createClient2 } = await import('@libsql/client/http');
-    const authToken = process.env[turso.authTokenEnv];
+    const authToken = turso.authToken ?? process.env[turso.authTokenEnv];
     const client2 = createClient2({
       url: remoteUrl,
       authToken
@@ -17761,6 +17849,12 @@ async function createVectorStore(config, cwd) {
       dimension: config.vector.dimension
     });
   }
+  if (isServerless()) {
+    throw new SearchSocketError(
+      "VECTOR_BACKEND_UNAVAILABLE",
+      `No remote vector database URL found (checked vector.turso.url and env var "${turso.urlEnv}"). Local SQLite storage is not available in serverless environments. Set ${turso.urlEnv} or pass vector.turso.url directly.`
+    );
+  }
   const { createClient } = await import('@libsql/client');
   const localPath = path__default.default.resolve(cwd, turso.localPath);
   fs__default.default.mkdirSync(path__default.default.dirname(localPath), { recursive: true });
@@ -17918,7 +18012,7 @@ var SearchEngine = class _SearchEngine {
     const groupByPage = (input.groupBy ?? "page") === "page";
     const candidateK = groupByPage ? Math.max(topK * 10, 50) : Math.max(50, topK);
     const embedStart = process.hrtime.bigint();
-    const queryEmbeddings = await this.embeddings.embedTexts([input.q], this.config.embeddings.model);
+    const queryEmbeddings = await this.embeddings.embedTexts([input.q], this.config.embeddings.model, "retrieval.query");
     const queryVector = queryEmbeddings[0];
     if (!queryVector || queryVector.length === 0 || queryVector.some((value) => !Number.isFinite(value))) {
       throw new SearchSocketError("VECTOR_BACKEND_UNAVAILABLE", "Unable to create query embedding.");
@@ -17946,13 +18040,17 @@ var SearchEngine = class _SearchEngine {
       usedRerank = true;
     }
     let results;
+    const minScore = this.config.ranking.minScore;
     if (groupByPage) {
-      const pages = aggregateByPage(ordered, this.config);
+      let pages = aggregateByPage(ordered, this.config);
+      if (minScore > 0) {
+        pages = pages.filter((p) => p.pageScore >= minScore);
+      }
       const minRatio = this.config.ranking.minChunkScoreRatio;
       results = pages.slice(0, topK).map((page) => {
         const bestScore = page.bestChunk.finalScore;
-        const minScore = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
-        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minScore).slice(0, 5);
+        const minScore2 = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
+        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minScore2).slice(0, 5);
         return {
           url: page.url,
           title: page.title,
@@ -17969,6 +18067,9 @@ var SearchEngine = class _SearchEngine {
         };
       });
     } else {
+      if (minScore > 0) {
+        ordered = ordered.filter((entry) => entry.finalScore >= minScore);
+      }
       results = ordered.slice(0, topK).map(({ hit, finalScore }) => ({
         url: hit.metadata.url,
         title: hit.metadata.title,
@@ -18040,43 +18141,67 @@ var SearchEngine = class _SearchEngine {
     }
   }
   async rerankHits(query, ranked, topK) {
-    if (this.config.rerank.provider !== "jina") {
+    if (!this.config.rerank.enabled) {
       throw new SearchSocketError(
         "INVALID_REQUEST",
-        "rerank=true requested but rerank.provider is not configured as 'jina'.",
+        "rerank=true requested but rerank.enabled is not set to true.",
         400
       );
     }
     if (!this.reranker) {
       throw new SearchSocketError(
         "CONFIG_MISSING",
-        `rerank=true requested but ${this.config.rerank.jina.apiKeyEnv} is not set.`,
+        `rerank=true requested but ${this.config.embeddings.apiKeyEnv} is not set.`,
         400
       );
     }
-    const candidates = ranked.map(({ hit }) => ({
-      id: hit.id,
-      text: [hit.metadata.title, hit.metadata.sectionTitle, hit.metadata.snippet].filter(Boolean).join("\n")
-    }));
+    const pageGroups = /* @__PURE__ */ new Map();
+    for (const entry of ranked) {
+      const url = entry.hit.metadata.url;
+      const group = pageGroups.get(url);
+      if (group) group.push(entry);
+      else pageGroups.set(url, [entry]);
+    }
+    const MAX_CHUNKS_PER_PAGE = 5;
+    const MIN_CHUNKS_PER_PAGE = 1;
+    const MIN_CHUNK_SCORE_RATIO = 0.5;
+    const pageCandidates = [];
+    for (const [url, chunks] of pageGroups) {
+      const byScore = [...chunks].sort((a, b) => b.finalScore - a.finalScore);
+      const bestScore = byScore[0].finalScore;
+      const scoreFloor = Number.isFinite(bestScore) ? bestScore * MIN_CHUNK_SCORE_RATIO : Number.NEGATIVE_INFINITY;
+      const selected = byScore.filter(
+        (c, i) => i < MIN_CHUNKS_PER_PAGE || c.finalScore >= scoreFloor
+      ).slice(0, MAX_CHUNKS_PER_PAGE);
+      selected.sort((a, b) => (a.hit.metadata.ordinal ?? 0) - (b.hit.metadata.ordinal ?? 0));
+      const first = selected[0].hit.metadata;
+      const parts = [first.title];
+      if (first.description) {
+        parts.push(first.description);
+      }
+      if (first.keywords && first.keywords.length > 0) {
+        parts.push(first.keywords.join(", "));
+      }
+      const body = selected.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
+      parts.push(body);
+      pageCandidates.push({ id: url, text: parts.join("\n\n") });
+    }
     const reranked = await this.reranker.rerank(
       query,
-      candidates,
+      pageCandidates,
       Math.max(topK, this.config.rerank.topN)
     );
-    const rerankScoreById = new Map(reranked.map((entry) => [entry.id, entry.score]));
+    const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {
-      const rerankScore = rerankScoreById.get(entry.hit.id);
-      const safeBaseScore = Number.isFinite(entry.finalScore) ? entry.finalScore : Number.NEGATIVE_INFINITY;
-      if (rerankScore === void 0 || !Number.isFinite(rerankScore)) {
-        return {
-          ...entry,
-          finalScore: safeBaseScore
-        };
+      const pageScore = scoreByUrl.get(entry.hit.metadata.url);
+      const base = Number.isFinite(entry.finalScore) ? entry.finalScore : Number.NEGATIVE_INFINITY;
+      if (pageScore === void 0 || !Number.isFinite(pageScore)) {
+        return { ...entry, finalScore: base };
       }
-      const combinedScore = rerankScore * this.config.ranking.weights.rerank + safeBaseScore * 1e-3;
+      const combined = pageScore * this.config.ranking.weights.rerank + base * 1e-3;
       return {
         ...entry,
-        finalScore: Number.isFinite(combinedScore) ? combinedScore : safeBaseScore
+        finalScore: Number.isFinite(combined) ? combined : base
       };
     }).sort((a, b) => {
       const delta = b.finalScore - a.finalScore;
@@ -18116,13 +18241,21 @@ function searchsocketHandle(options = {}) {
   let rateLimiter = null;
   const getConfig = async () => {
     if (!configPromise) {
-      const configP = options.config ? Promise.resolve(options.config) : loadConfig({
-        cwd: options.cwd,
-        configPath: options.configPath
-      });
+      let configP;
+      if (options.config) {
+        configP = Promise.resolve(options.config);
+      } else if (options.rawConfig) {
+        const cwd = options.cwd ?? process.cwd();
+        configP = Promise.resolve(mergeConfig(cwd, options.rawConfig));
+      } else {
+        configP = loadConfig({
+          cwd: options.cwd,
+          configPath: options.configPath
+        });
+      }
       configPromise = configP.then((config) => {
         apiPath = apiPath ?? config.api.path;
-        if (config.api.rateLimit) {
+        if (config.api.rateLimit && !isServerless()) {
           rateLimiter = new InMemoryRateLimiter(config.api.rateLimit.windowMs, config.api.rateLimit.max);
         }
         return config;
@@ -18132,10 +18265,9 @@ function searchsocketHandle(options = {}) {
   };
   const getEngine = async () => {
     if (!enginePromise) {
-      const config = options.config;
+      const config = await getConfig();
       enginePromise = SearchEngine.create({
         cwd: options.cwd,
-        configPath: options.configPath,
         config
       });
     }
@@ -18562,7 +18694,9 @@ function chunkMirrorPage(page, config, scope) {
       incomingLinks: page.incomingLinks,
       routeFile: page.routeFile,
       tags: page.tags,
-      contentHash: ""
+      contentHash: "",
+      description: page.description,
+      keywords: page.keywords
     };
     const embeddingText = buildEmbeddingText(summaryChunk, config.chunking.prependTitle);
     summaryChunk.contentHash = sha256(normalizeText(embeddingText));
@@ -18589,7 +18723,9 @@ function chunkMirrorPage(page, config, scope) {
       incomingLinks: page.incomingLinks,
       routeFile: page.routeFile,
       tags: page.tags,
-      contentHash: ""
+      contentHash: "",
+      description: page.description,
+      keywords: page.keywords
     };
     const embeddingText = buildEmbeddingText(chunk, config.chunking.prependTitle);
     chunk.contentHash = sha256(normalizeText(embeddingText));
@@ -19670,14 +19806,16 @@ function mapUrlToRoute(urlPath, patterns) {
 var Logger = class {
   json;
   verbose;
+  quiet;
   stderrOnly;
   constructor(opts = {}) {
     this.json = opts.json ?? false;
     this.verbose = opts.verbose ?? false;
+    this.quiet = opts.quiet ?? false;
     this.stderrOnly = opts.stderrOnly ?? false;
   }
   info(message) {
-    if (this.json) {
+    if (this.quiet || this.json) {
       return;
     }
     this.writeOut(`${message}
@@ -19691,7 +19829,7 @@ var Logger = class {
       this.logJson("debug", { message });
       return;
     }
-    this.writeOut(`${message}
+    this.writeOut(`  ${message}
 `);
   }
   warn(message) {
@@ -19718,7 +19856,7 @@ var Logger = class {
       this.logJson(event, data);
       return;
     }
-    this.writeOut(`[${event}] ${data ? JSON.stringify(data) : ""}
+    this.writeOut(`  [${event}] ${data ? JSON.stringify(data) : ""}
 `);
   }
   writeOut(text) {
@@ -19903,11 +20041,108 @@ async function startPreviewServer(cwd, options, logger3) {
 
 // src/indexing/sources/build/index.ts
 var logger = new Logger();
+function extractLinksFromHtml(html, pageUrl, baseOrigin) {
+  const $ = cheerio.load(html);
+  const links = [];
+  $("a[href]").each((_i, el) => {
+    const href = $(el).attr("href");
+    if (!href || href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) {
+      return;
+    }
+    try {
+      const resolved = new URL(href, `${baseOrigin}${pageUrl}`);
+      if (resolved.origin !== baseOrigin) return;
+      if (!["http:", "https:"].includes(resolved.protocol)) return;
+      links.push(normalizeUrlPath(resolved.pathname));
+    } catch {
+    }
+  });
+  return [...new Set(links)];
+}
+async function discoverPages(server, buildConfig, pipelineMaxPages) {
+  const { seedUrls, maxDepth, exclude } = buildConfig;
+  const baseOrigin = new URL(server.baseUrl).origin;
+  let effectiveMax = buildConfig.maxPages;
+  if (typeof pipelineMaxPages === "number") {
+    const floored = Math.max(0, Math.floor(pipelineMaxPages));
+    effectiveMax = Math.min(effectiveMax, floored);
+  }
+  if (effectiveMax === 0) return [];
+  const visited = /* @__PURE__ */ new Set();
+  const pages = [];
+  const queue = [];
+  const limit = pLimit2__default.default(8);
+  for (const seed of seedUrls) {
+    const normalized = normalizeUrlPath(seed);
+    if (!visited.has(normalized) && !isExcluded(normalized, exclude)) {
+      visited.add(normalized);
+      queue.push({ url: normalized, depth: 0 });
+    }
+  }
+  while (queue.length > 0 && pages.length < effectiveMax) {
+    const remaining = effectiveMax - pages.length;
+    const batch = queue.splice(0, remaining);
+    const results = await Promise.allSettled(
+      batch.map(
+        (item) => limit(async () => {
+          const fullUrl = joinUrl(server.baseUrl, item.url);
+          const response = await fetch(fullUrl);
+          if (!response.ok) {
+            logger.warn(`Skipping ${item.url}: ${response.status} ${response.statusText}`);
+            return null;
+          }
+          const contentType = response.headers.get("content-type") ?? "";
+          if (!contentType.includes("text/html")) {
+            return null;
+          }
+          const html = await response.text();
+          if (item.depth < maxDepth) {
+            const links = extractLinksFromHtml(html, item.url, baseOrigin);
+            for (const link of links) {
+              if (!visited.has(link) && !isExcluded(link, exclude)) {
+                visited.add(link);
+                queue.push({ url: link, depth: item.depth + 1 });
+              }
+            }
+          }
+          return {
+            url: item.url,
+            html,
+            sourcePath: fullUrl,
+            outgoingLinks: []
+          };
+        })
+      )
+    );
+    for (const result of results) {
+      if (result.status === "fulfilled" && result.value) {
+        pages.push(result.value);
+      }
+    }
+  }
+  if (pages.length >= effectiveMax && queue.length > 0) {
+    logger.warn(`Discovery crawl reached maxPages limit (${effectiveMax}), ${queue.length} URLs not visited.`);
+  }
+  logger.event("build_discover_complete", {
+    pagesFound: pages.length,
+    urlsVisited: visited.size,
+    urlsSkipped: queue.length
+  });
+  return pages;
+}
 async function loadBuildPages(cwd, config, maxPages) {
   const buildConfig = config.source.build;
   if (!buildConfig) {
     throw new Error("build source config is missing");
   }
+  if (buildConfig.discover) {
+    const server2 = await startPreviewServer(cwd, { previewTimeout: buildConfig.previewTimeout }, logger);
+    try {
+      return await discoverPages(server2, buildConfig, maxPages);
+    } finally {
+      await server2.shutdown();
+    }
+  }
   const routes = await parseManifest(cwd, buildConfig.outputDir);
   const expanded = expandRoutes(routes, buildConfig.paramValues, buildConfig.exclude, logger);
   logger.event("build_routes_discovered", {
@@ -19918,7 +20153,7 @@ async function loadBuildPages(cwd, config, maxPages) {
   const selected = typeof maxCount === "number" ? expanded.slice(0, maxCount) : expanded;
   const server = await startPreviewServer(cwd, { previewTimeout: buildConfig.previewTimeout }, logger);
   try {
-    const concurrencyLimit = pLimit__default.default(8);
+    const concurrencyLimit = pLimit2__default.default(8);
     const results = await Promise.allSettled(
       selected.map(
         (route) => concurrencyLimit(async () => {
@@ -20087,7 +20322,7 @@ async function loadCrawledPages(config, maxPages) {
   const routes = await resolveRoutes(config);
   const maxCount = typeof maxPages === "number" ? Math.max(0, Math.floor(maxPages)) : void 0;
   const selected = typeof maxCount === "number" ? routes.slice(0, maxCount) : routes;
-  const concurrencyLimit = pLimit__default.default(8);
+  const concurrencyLimit = pLimit2__default.default(8);
   const results = await Promise.allSettled(
     selected.map(
       (route) => concurrencyLimit(async () => {
@@ -20141,9 +20376,7 @@ async function loadStaticOutputPages(cwd, config, maxPages) {
 
 // src/indexing/pipeline.ts
 var EMBEDDING_PRICE_PER_1K_TOKENS_USD = {
-  "text-embedding-3-small": 2e-5,
-  "text-embedding-3-large": 13e-5,
-  "text-embedding-ada-002": 1e-4
+  "jina-embeddings-v3": 2e-5
 };
 var DEFAULT_EMBEDDING_PRICE_PER_1K = 2e-5;
 var IndexPipeline = class _IndexPipeline {
@@ -20189,9 +20422,15 @@ var IndexPipeline = class _IndexPipeline {
     };
     const scope = resolveScope(this.config, options.scopeOverride);
     const { statePath } = ensureStateDirs(this.cwd, this.config.state.dir, scope);
+    const sourceMode = options.sourceOverride ?? this.config.source.mode;
+    this.logger.info(`Indexing scope "${scope.scopeName}" (source: ${sourceMode}, model: ${this.config.embeddings.model})`);
     if (options.force) {
+      this.logger.info("Force mode enabled \u2014 full rebuild");
       await cleanMirrorForScope(statePath, scope);
     }
+    if (options.dryRun) {
+      this.logger.info("Dry run \u2014 no writes will be performed");
+    }
     const manifestStart = stageStart();
     const existingHashes = await this.vectorStore.getContentHashes(scope);
     const existingModelId = await this.vectorStore.getScopeModelId(scope);
@@ -20202,8 +20441,9 @@ var IndexPipeline = class _IndexPipeline {
       );
     }
     stageEnd("manifest", manifestStart);
+    this.logger.debug(`Manifest: ${existingHashes.size} existing chunk hashes loaded`);
     const sourceStart = stageStart();
-    const sourceMode = options.sourceOverride ?? this.config.source.mode;
+    this.logger.info(`Loading pages (source: ${sourceMode})...`);
     let sourcePages;
     if (sourceMode === "static-output") {
       sourcePages = await loadStaticOutputPages(this.cwd, this.config, options.maxPages);
@@ -20215,10 +20455,13 @@ var IndexPipeline = class _IndexPipeline {
       sourcePages = await loadContentFilesPages(this.cwd, this.config, options.maxPages);
     }
     stageEnd("source", sourceStart);
+    this.logger.info(`Loaded ${sourcePages.length} page${sourcePages.length === 1 ? "" : "s"} (${stageTimingsMs["source"]}ms)`);
     const routeStart = stageStart();
     const routePatterns = await buildRoutePatterns(this.cwd);
     stageEnd("route_map", routeStart);
+    this.logger.debug(`Route mapping: ${routePatterns.length} pattern${routePatterns.length === 1 ? "" : "s"} discovered (${stageTimingsMs["route_map"]}ms)`);
     const extractStart = stageStart();
+    this.logger.info("Extracting content...");
     const extractedPages = [];
     for (const sourcePage of sourcePages) {
       const extracted = sourcePage.html ? extractFromHtml(sourcePage.url, sourcePage.html, this.config) : extractFromMarkdown(sourcePage.url, sourcePage.markdown ?? "", sourcePage.title);
@@ -20247,6 +20490,8 @@ var IndexPipeline = class _IndexPipeline {
       uniquePages.push(page);
     }
     stageEnd("extract", extractStart);
+    const skippedPages = sourcePages.length - uniquePages.length;
+    this.logger.info(`Extracted ${uniquePages.length} page${uniquePages.length === 1 ? "" : "s"}${skippedPages > 0 ? ` (${skippedPages} skipped)` : ""} (${stageTimingsMs["extract"]}ms)`);
     const linkStart = stageStart();
     const pageSet = new Set(uniquePages.map((page) => normalizeUrlPath(page.url)));
     const incomingLinkCount = /* @__PURE__ */ new Map();
@@ -20262,7 +20507,9 @@ var IndexPipeline = class _IndexPipeline {
       }
     }
     stageEnd("links", linkStart);
+    this.logger.debug(`Link analysis: computed incoming links for ${incomingLinkCount.size} pages (${stageTimingsMs["links"]}ms)`);
     const mirrorStart = stageStart();
+    this.logger.info("Writing mirror pages...");
     const mirrorPages = [];
     let routeExact = 0;
     let routeBestEffort = 0;
@@ -20332,7 +20579,9 @@ var IndexPipeline = class _IndexPipeline {
       await this.vectorStore.upsertPages(pageRecords, scope);
     }
     stageEnd("mirror", mirrorStart);
+    this.logger.info(`Mirrored ${mirrorPages.length} page${mirrorPages.length === 1 ? "" : "s"} (${routeExact} exact, ${routeBestEffort} best-effort) (${stageTimingsMs["mirror"]}ms)`);
     const chunkStart = stageStart();
+    this.logger.info("Chunking pages...");
     let chunks = mirrorPages.flatMap((page) => chunkMirrorPage(page, this.config, scope));
     const maxChunks = typeof options.maxChunks === "number" ? Math.max(0, Math.floor(options.maxChunks)) : void 0;
     if (typeof maxChunks === "number") {
@@ -20345,6 +20594,7 @@ var IndexPipeline = class _IndexPipeline {
       });
     }
     stageEnd("chunk", chunkStart);
+    this.logger.info(`Chunked into ${chunks.length} chunk${chunks.length === 1 ? "" : "s"} (${stageTimingsMs["chunk"]}ms)`);
     const currentChunkMap = /* @__PURE__ */ new Map();
     for (const chunk of chunks) {
       currentChunkMap.set(chunk.chunkKey, chunk);
@@ -20363,6 +20613,7 @@ var IndexPipeline = class _IndexPipeline {
       return existingHash !== chunk.contentHash;
     });
     const deletes = [...existingHashes.keys()].filter((chunkKey) => !currentChunkMap.has(chunkKey));
+    this.logger.info(`Changes detected: ${changedChunks.length} changed, ${deletes.length} deleted, ${chunks.length - changedChunks.length} unchanged`);
     const embedStart = stageStart();
     const chunkTokenEstimates = /* @__PURE__ */ new Map();
     for (const chunk of changedChunks) {
@@ -20377,9 +20628,11 @@ var IndexPipeline = class _IndexPipeline {
     let newEmbeddings = 0;
     const vectorsByChunk = /* @__PURE__ */ new Map();
     if (!options.dryRun && changedChunks.length > 0) {
+      this.logger.info(`Embedding ${changedChunks.length} chunk${changedChunks.length === 1 ? "" : "s"} (~${estimatedTokens.toLocaleString()} tokens, ~$${estimatedCostUSD.toFixed(6)})...`);
       const embeddings = await this.embeddings.embedTexts(
         changedChunks.map((chunk) => buildEmbeddingText(chunk, this.config.chunking.prependTitle)),
-        this.config.embeddings.model
+        this.config.embeddings.model,
+        "retrieval.passage"
       );
       if (embeddings.length !== changedChunks.length) {
         throw new SearchSocketError(
@@ -20402,8 +20655,14 @@ var IndexPipeline = class _IndexPipeline {
       }
     }
     stageEnd("embedding", embedStart);
+    if (changedChunks.length > 0) {
+      this.logger.info(`Embedded ${newEmbeddings} chunk${newEmbeddings === 1 ? "" : "s"} (${stageTimingsMs["embedding"]}ms)`);
+    } else {
+      this.logger.info("No chunks to embed \u2014 all up to date");
+    }
     const syncStart = stageStart();
     if (!options.dryRun) {
+      this.logger.info("Syncing vectors...");
       const upserts = [];
       for (const chunk of changedChunks) {
         const vector = vectorsByChunk.get(chunk.chunkKey);
@@ -20422,12 +20681,16 @@ var IndexPipeline = class _IndexPipeline {
             sectionTitle: chunk.sectionTitle ?? "",
             headingPath: chunk.headingPath,
             snippet: chunk.snippet,
+            chunkText: chunk.chunkText.slice(0, 4e3),
+            ordinal: chunk.ordinal,
             contentHash: chunk.contentHash,
             modelId: this.config.embeddings.model,
             depth: chunk.depth,
             incomingLinks: chunk.incomingLinks,
             routeFile: chunk.routeFile,
-            tags: chunk.tags
+            tags: chunk.tags,
+            description: chunk.description,
+            keywords: chunk.keywords
           }
         });
       }
@@ -20441,6 +20704,7 @@ var IndexPipeline = class _IndexPipeline {
       }
     }
     stageEnd("sync", syncStart);
+    this.logger.debug(`Sync complete (${stageTimingsMs["sync"]}ms)`);
     const finalizeStart = stageStart();
     if (!options.dryRun) {
       const scopeInfo = {
@@ -20460,6 +20724,7 @@ var IndexPipeline = class _IndexPipeline {
       });
     }
     stageEnd("finalize", finalizeStart);
+    this.logger.info("Done.");
     return {
       pagesProcessed: mirrorPages.length,
       chunksTotal: chunks.length,