searchsocket 0.2.0 → 0.3.0

package/dist/sveltekit.js

@@ -2,8 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import { createJiti } from 'jiti';
 import { z } from 'zod';
-import OpenAI from 'openai';
-import pLimit from 'p-limit';
+import pLimit2 from 'p-limit';
 import { execSync, spawn } from 'child_process';
 import { createHash } from 'crypto';
 import { load } from 'cheerio';
@@ -16616,7 +16615,11 @@ var searchSocketConfigSchema = z.object({
       outputDir: z.string().min(1).optional(),
       paramValues: z.record(z.string(), z.array(z.string())).optional(),
       exclude: z.array(z.string()).optional(),
-      previewTimeout: z.number().int().positive().optional()
+      previewTimeout: z.number().int().positive().optional(),
+      discover: z.boolean().optional(),
+      seedUrls: z.array(z.string()).optional(),
+      maxPages: z.number().int().positive().optional(),
+      maxDepth: z.number().int().nonnegative().optional()
     }).optional()
   }).optional(),
   extract: z.object({
@@ -16643,8 +16646,9 @@ var searchSocketConfigSchema = z.object({
     pageSummaryChunk: z.boolean().optional()
   }).optional(),
   embeddings: z.object({
-    provider: z.literal("openai").optional(),
+    provider: z.literal("jina").optional(),
     model: z.string().min(1).optional(),
+    apiKey: z.string().min(1).optional(),
     apiKeyEnv: z.string().min(1).optional(),
     batchSize: z.number().int().positive().optional(),
     concurrency: z.number().int().positive().optional(),
@@ -16653,18 +16657,17 @@ var searchSocketConfigSchema = z.object({
   vector: z.object({
     dimension: z.number().int().positive().optional(),
     turso: z.object({
+      url: z.string().url().optional(),
+      authToken: z.string().min(1).optional(),
       urlEnv: z.string().optional(),
       authTokenEnv: z.string().optional(),
       localPath: z.string().optional()
     }).optional()
   }).optional(),
   rerank: z.object({
-    provider: z.enum(["none", "jina"]).optional(),
+    enabled: z.boolean().optional(),
     topN: z.number().int().positive().optional(),
-    jina: z.object({
-      apiKeyEnv: z.string().optional(),
-      model: z.string().optional()
-    }).optional()
+    model: z.string().optional()
   }).optional(),
   ranking: z.object({
     enableIncomingLinkBoost: z.boolean().optional(),
@@ -16673,6 +16676,7 @@ var searchSocketConfigSchema = z.object({
     aggregationCap: z.number().int().positive().optional(),
     aggregationDecay: z.number().min(0).max(1).optional(),
     minChunkScoreRatio: z.number().min(0).max(1).optional(),
+    minScore: z.number().min(0).max(1).optional(),
     weights: z.object({
       incomingLinks: z.number().optional(),
       depth: z.number().optional(),
@@ -16753,9 +16757,9 @@ function createDefaultConfig(projectId) {
       pageSummaryChunk: true
     },
     embeddings: {
-      provider: "openai",
-      model: "text-embedding-3-small",
-      apiKeyEnv: "OPENAI_API_KEY",
+      provider: "jina",
+      model: "jina-embeddings-v3",
+      apiKeyEnv: "JINA_API_KEY",
       batchSize: 64,
       concurrency: 4
     },
@@ -16767,12 +16771,9 @@ function createDefaultConfig(projectId) {
       }
     },
     rerank: {
-      provider: "none",
+      enabled: false,
       topN: 20,
-      jina: {
-        apiKeyEnv: "JINA_API_KEY",
-        model: "jina-reranker-v2-base-multilingual"
-      }
+      model: "jina-reranker-v2-base-multilingual"
     },
     ranking: {
       enableIncomingLinkBoost: true,
@@ -16781,6 +16782,7 @@ function createDefaultConfig(projectId) {
       aggregationCap: 5,
       aggregationDecay: 0.5,
       minChunkScoreRatio: 0.5,
+      minScore: 0,
       weights: {
         incomingLinks: 0.05,
         depth: 0.03,
@@ -16907,7 +16909,11 @@ ${issues}`
         outputDir: parsed.source.build.outputDir ?? ".svelte-kit/output",
         paramValues: parsed.source.build.paramValues ?? {},
         exclude: parsed.source.build.exclude ?? [],
-        previewTimeout: parsed.source.build.previewTimeout ?? 3e4
+        previewTimeout: parsed.source.build.previewTimeout ?? 3e4,
+        discover: parsed.source.build.discover ?? false,
+        seedUrls: parsed.source.build.seedUrls ?? ["/"],
+        maxPages: parsed.source.build.maxPages ?? 200,
+        maxDepth: parsed.source.build.maxDepth ?? 10
       } : void 0
     },
     extract: {
@@ -16936,11 +16942,7 @@ ${issues}`
     },
     rerank: {
       ...defaults.rerank,
-      ...parsed.rerank,
-      jina: {
-        ...defaults.rerank.jina,
-        ...parsed.rerank?.jina
-      }
+      ...parsed.rerank
     },
     ranking: {
       ...defaults.ranking,
@@ -16987,7 +16989,11 @@ ${issues}`
       outputDir: ".svelte-kit/output",
       paramValues: {},
       exclude: [],
-      previewTimeout: 3e4
+      previewTimeout: 3e4,
+      discover: false,
+      seedUrls: ["/"],
+      maxPages: 200,
+      maxDepth: 10
     };
   }
   if (merged.source.mode === "crawl" && !merged.source.crawl?.baseUrl) {
@@ -17022,15 +17028,21 @@ async function loadConfig(options = {}) {
   const raw = loaded.default ?? loaded;
   return mergeConfig(cwd, raw);
 }
+
+// src/core/serverless.ts
+function isServerless() {
+  return !!(process.env.VERCEL || process.env.NETLIFY || process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.FUNCTIONS_WORKER || process.env.CF_PAGES);
+}
 function sleep(ms) {
   return new Promise((resolve) => {
     setTimeout(resolve, ms);
   });
 }
-var OpenAIEmbeddingsProvider = class {
-  client;
+var JinaEmbeddingsProvider = class {
+  apiKey;
   batchSize;
   concurrency;
+  defaultTask;
   constructor(options) {
     if (!Number.isInteger(options.batchSize) || options.batchSize <= 0) {
       throw new Error(`Invalid batchSize: ${options.batchSize}. batchSize must be a positive integer.`);
@@ -17038,11 +17050,10 @@ var OpenAIEmbeddingsProvider = class {
     if (!Number.isInteger(options.concurrency) || options.concurrency <= 0) {
       throw new Error(`Invalid concurrency: ${options.concurrency}. concurrency must be a positive integer.`);
     }
-    this.client = new OpenAI({
-      apiKey: options.apiKey
-    });
+    this.apiKey = options.apiKey;
     this.batchSize = options.batchSize;
     this.concurrency = options.concurrency;
+    this.defaultTask = options.task ?? "retrieval.passage";
   }
   estimateTokens(text) {
     const normalized = text.trim();
@@ -17056,7 +17067,7 @@ var OpenAIEmbeddingsProvider = class {
     const lexicalEstimate = Math.ceil(wordCount * 1.25 + punctuationCount * 0.45 + cjkCount * 1.6);
     return Math.max(1, Math.max(charEstimate, lexicalEstimate));
   }
-  async embedTexts(texts, modelId) {
+  async embedTexts(texts, modelId, task) {
     if (texts.length === 0) {
       return [];
     }
@@ -17068,37 +17079,56 @@ var OpenAIEmbeddingsProvider = class {
       });
     }
     const outputs = new Array(batches.length);
-    const limit = pLimit(this.concurrency);
+    const limit = pLimit2(this.concurrency);
     await Promise.all(
       batches.map(
         (batch, position) => limit(async () => {
-          outputs[position] = await this.embedWithRetry(batch.values, modelId);
+          outputs[position] = await this.embedWithRetry(batch.values, modelId, task ?? this.defaultTask);
         })
       )
     );
     return outputs.flat();
   }
-  async embedWithRetry(texts, modelId) {
+  async embedWithRetry(texts, modelId, task) {
     const maxAttempts = 5;
     let attempt = 0;
     while (attempt < maxAttempts) {
       attempt += 1;
+      let response;
       try {
-        const response = await this.client.embeddings.create({
-          model: modelId,
-          input: texts,
-          encoding_format: "float"
+        response = await fetch("https://api.jina.ai/v1/embeddings", {
+          method: "POST",
+          headers: {
+            "content-type": "application/json",
+            authorization: `Bearer ${this.apiKey}`
+          },
+          body: JSON.stringify({
+            model: modelId,
+            input: texts,
+            task
+          })
         });
-        return response.data.map((entry) => entry.embedding);
       } catch (error) {
-        const status = error.status;
-        const retryable = status === 429 || typeof status === "number" && status >= 500;
-        if (!retryable || attempt >= maxAttempts) {
+        if (attempt >= maxAttempts) {
           throw error;
         }
-        const delay = Math.min(2 ** attempt * 300, 5e3);
-        await sleep(delay);
+        await sleep(Math.min(2 ** attempt * 300, 5e3));
+        continue;
+      }
+      if (!response.ok) {
+        const retryable = response.status === 429 || response.status >= 500;
+        if (!retryable || attempt >= maxAttempts) {
+          const errorBody = await response.text();
+          throw new Error(`Jina embeddings failed (${response.status}): ${errorBody}`);
+        }
+        await sleep(Math.min(2 ** attempt * 300, 5e3));
+        continue;
       }
+      const payload = await response.json();
+      if (!payload.data || !Array.isArray(payload.data)) {
+        throw new Error("Invalid Jina embeddings response format");
+      }
+      return payload.data.map((entry) => entry.embedding);
     }
     throw new Error("Unreachable retry state");
   }
@@ -17106,20 +17136,20 @@ var OpenAIEmbeddingsProvider = class {
 
 // src/embeddings/factory.ts
 function createEmbeddingsProvider(config) {
-  if (config.embeddings.provider !== "openai") {
+  if (config.embeddings.provider !== "jina") {
     throw new SearchSocketError(
       "CONFIG_MISSING",
       `Unsupported embeddings provider ${config.embeddings.provider}`
     );
   }
-  const apiKey = process.env[config.embeddings.apiKeyEnv];
+  const apiKey = config.embeddings.apiKey ?? process.env[config.embeddings.apiKeyEnv];
   if (!apiKey) {
     throw new SearchSocketError(
       "CONFIG_MISSING",
-      `Missing embeddings API key env var: ${config.embeddings.apiKeyEnv}`
+      `Missing embeddings API key: provide embeddings.apiKey or set env var ${config.embeddings.apiKeyEnv}`
     );
   }
-  return new OpenAIEmbeddingsProvider({
+  return new JinaEmbeddingsProvider({
     apiKey,
     batchSize: config.embeddings.batchSize,
     concurrency: config.embeddings.concurrency
@@ -17282,20 +17312,17 @@ var JinaReranker = class {
 
 // src/rerank/factory.ts
 function createReranker(config) {
-  if (config.rerank.provider === "none") {
+  if (!config.rerank.enabled) {
     return null;
   }
-  if (config.rerank.provider === "jina") {
-    const apiKey = process.env[config.rerank.jina.apiKeyEnv];
-    if (!apiKey) {
-      return null;
-    }
-    return new JinaReranker({
-      apiKey,
-      model: config.rerank.jina.model
-    });
+  const apiKey = config.embeddings.apiKey ?? process.env[config.embeddings.apiKeyEnv];
+  if (!apiKey) {
+    return null;
   }
-  return null;
+  return new JinaReranker({
+    apiKey,
+    model: config.rerank.model
+  });
 }
 
 // src/utils/time.ts
@@ -17400,6 +17427,16 @@ var TursoVectorStore = class {
   }
   async ensureChunks(dim) {
     if (this.chunksReady) return;
+    const exists = await this.chunksTableExists();
+    if (exists) {
+      const currentDim = await this.getChunksDimension();
+      if (currentDim !== null && currentDim !== dim) {
+        await this.client.batch([
+          "DROP INDEX IF EXISTS idx",
+          "DROP TABLE IF EXISTS chunks"
+        ]);
+      }
+    }
     await this.client.batch([
       `CREATE TABLE IF NOT EXISTS chunks (
         id             TEXT PRIMARY KEY,
@@ -17411,6 +17448,8 @@ var TursoVectorStore = class {
         section_title  TEXT NOT NULL DEFAULT '',
         heading_path   TEXT NOT NULL DEFAULT '[]',
         snippet        TEXT NOT NULL DEFAULT '',
+        chunk_text     TEXT NOT NULL DEFAULT '',
+        ordinal        INTEGER NOT NULL DEFAULT 0,
         content_hash   TEXT NOT NULL DEFAULT '',
         model_id       TEXT NOT NULL DEFAULT '',
         depth          INTEGER NOT NULL DEFAULT 0,
@@ -17421,6 +17460,19 @@ var TursoVectorStore = class {
       )`,
       `CREATE INDEX IF NOT EXISTS idx ON chunks (libsql_vector_idx(embedding, 'metric=cosine'))`
     ]);
+    const chunkMigrationCols = [
+      { name: "chunk_text", def: "TEXT NOT NULL DEFAULT ''" },
+      { name: "ordinal", def: "INTEGER NOT NULL DEFAULT 0" }
+    ];
+    for (const col of chunkMigrationCols) {
+      try {
+        await this.client.execute(`ALTER TABLE chunks ADD COLUMN ${col.name} ${col.def}`);
+      } catch (error) {
+        if (error instanceof Error && !error.message.includes("duplicate column")) {
+          throw error;
+        }
+      }
+    }
     this.chunksReady = true;
   }
   async ensurePages() {
@@ -17455,6 +17507,38 @@ var TursoVectorStore = class {
       throw error;
     }
   }
+  /**
+   * Read the current F32_BLOB dimension from the chunks table schema.
+   * Returns null if the table doesn't exist or the dimension can't be parsed.
+   */
+  async getChunksDimension() {
+    try {
+      const rs = await this.client.execute(
+        "SELECT sql FROM sqlite_master WHERE type='table' AND name='chunks'"
+      );
+      if (rs.rows.length === 0) return null;
+      const sql = rs.rows[0].sql;
+      const match = sql.match(/F32_BLOB\((\d+)\)/i);
+      return match ? parseInt(match[1], 10) : null;
+    } catch {
+      return null;
+    }
+  }
+  /**
+   * Drop all SearchSocket tables (chunks, registry, pages) and their indexes.
+   * Used by `clean --remote` for a full reset.
+   */
+  async dropAllTables() {
+    await this.client.batch([
+      "DROP INDEX IF EXISTS idx",
+      "DROP TABLE IF EXISTS chunks",
+      "DROP TABLE IF EXISTS registry",
+      "DROP TABLE IF EXISTS pages"
+    ]);
+    this.chunksReady = false;
+    this.registryReady = false;
+    this.pagesReady = false;
+  }
   async upsert(records, _scope) {
     if (records.length === 0) return;
     const dim = this.dimension ?? records[0].vector.length;
@@ -17465,9 +17549,9 @@ var TursoVectorStore = class {
       const stmts = batch.map((r) => ({
         sql: `INSERT OR REPLACE INTO chunks
               (id, project_id, scope_name, url, path, title, section_title,
-               heading_path, snippet, content_hash, model_id, depth,
+               heading_path, snippet, chunk_text, ordinal, content_hash, model_id, depth,
                incoming_links, route_file, tags, embedding)
-              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector(?))`,
+              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector(?))`,
         args: [
           r.id,
           r.metadata.projectId,
@@ -17478,6 +17562,8 @@ var TursoVectorStore = class {
           r.metadata.sectionTitle,
           JSON.stringify(r.metadata.headingPath),
           r.metadata.snippet,
+          r.metadata.chunkText,
+          r.metadata.ordinal,
           r.metadata.contentHash,
           r.metadata.modelId,
           r.metadata.depth,
@@ -17496,7 +17582,8 @@ var TursoVectorStore = class {
     const queryJson = JSON.stringify(queryVector);
     const rs = await this.client.execute({
       sql: `SELECT c.id, c.project_id, c.scope_name, c.url, c.path, c.title,
-                   c.section_title, c.heading_path, c.snippet, c.content_hash,
+                   c.section_title, c.heading_path, c.snippet, c.chunk_text,
+                   c.ordinal, c.content_hash,
                    c.model_id, c.depth, c.incoming_links, c.route_file, c.tags,
                    vector_distance_cos(c.embedding, vector(?)) AS distance
             FROM vector_top_k('idx', vector(?), ?) AS v
@@ -17540,6 +17627,8 @@ var TursoVectorStore = class {
           sectionTitle: row.section_title,
           headingPath: JSON.parse(row.heading_path || "[]"),
           snippet: row.snippet,
+          chunkText: row.chunk_text || "",
+          ordinal: row.ordinal || 0,
           contentHash: row.content_hash,
           modelId: row.model_id,
           depth: row.depth,
@@ -17735,10 +17824,10 @@ var TursoVectorStore = class {
 // src/vector/factory.ts
 async function createVectorStore(config, cwd) {
   const turso = config.vector.turso;
-  const remoteUrl = process.env[turso.urlEnv];
+  const remoteUrl = turso.url ?? process.env[turso.urlEnv];
   if (remoteUrl) {
     const { createClient: createClient2 } = await import('@libsql/client/http');
-    const authToken = process.env[turso.authTokenEnv];
+    const authToken = turso.authToken ?? process.env[turso.authTokenEnv];
     const client2 = createClient2({
       url: remoteUrl,
       authToken
@@ -17748,6 +17837,12 @@ async function createVectorStore(config, cwd) {
       dimension: config.vector.dimension
     });
   }
+  if (isServerless()) {
+    throw new SearchSocketError(
+      "VECTOR_BACKEND_UNAVAILABLE",
+      `No remote vector database URL found (checked vector.turso.url and env var "${turso.urlEnv}"). Local SQLite storage is not available in serverless environments. Set ${turso.urlEnv} or pass vector.turso.url directly.`
+    );
+  }
   const { createClient } = await import('@libsql/client');
   const localPath = path.resolve(cwd, turso.localPath);
   fs.mkdirSync(path.dirname(localPath), { recursive: true });
@@ -17905,7 +18000,7 @@ var SearchEngine = class _SearchEngine {
     const groupByPage = (input.groupBy ?? "page") === "page";
     const candidateK = groupByPage ? Math.max(topK * 10, 50) : Math.max(50, topK);
     const embedStart = process.hrtime.bigint();
-    const queryEmbeddings = await this.embeddings.embedTexts([input.q], this.config.embeddings.model);
+    const queryEmbeddings = await this.embeddings.embedTexts([input.q], this.config.embeddings.model, "retrieval.query");
     const queryVector = queryEmbeddings[0];
     if (!queryVector || queryVector.length === 0 || queryVector.some((value) => !Number.isFinite(value))) {
       throw new SearchSocketError("VECTOR_BACKEND_UNAVAILABLE", "Unable to create query embedding.");
@@ -17933,13 +18028,17 @@ var SearchEngine = class _SearchEngine {
       usedRerank = true;
     }
     let results;
+    const minScore = this.config.ranking.minScore;
     if (groupByPage) {
-      const pages = aggregateByPage(ordered, this.config);
+      let pages = aggregateByPage(ordered, this.config);
+      if (minScore > 0) {
+        pages = pages.filter((p) => p.pageScore >= minScore);
+      }
       const minRatio = this.config.ranking.minChunkScoreRatio;
       results = pages.slice(0, topK).map((page) => {
         const bestScore = page.bestChunk.finalScore;
-        const minScore = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
-        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minScore).slice(0, 5);
+        const minScore2 = Number.isFinite(bestScore) ? bestScore * minRatio : Number.NEGATIVE_INFINITY;
+        const meaningful = page.matchingChunks.filter((c) => c.finalScore >= minScore2).slice(0, 5);
         return {
           url: page.url,
           title: page.title,
@@ -17956,6 +18055,9 @@ var SearchEngine = class _SearchEngine {
         };
       });
     } else {
+      if (minScore > 0) {
+        ordered = ordered.filter((entry) => entry.finalScore >= minScore);
+      }
       results = ordered.slice(0, topK).map(({ hit, finalScore }) => ({
         url: hit.metadata.url,
         title: hit.metadata.title,
@@ -18027,43 +18129,54 @@ var SearchEngine = class _SearchEngine {
     }
   }
   async rerankHits(query, ranked, topK) {
-    if (this.config.rerank.provider !== "jina") {
+    if (!this.config.rerank.enabled) {
       throw new SearchSocketError(
         "INVALID_REQUEST",
-        "rerank=true requested but rerank.provider is not configured as 'jina'.",
+        "rerank=true requested but rerank.enabled is not set to true.",
         400
       );
     }
     if (!this.reranker) {
       throw new SearchSocketError(
         "CONFIG_MISSING",
-        `rerank=true requested but ${this.config.rerank.jina.apiKeyEnv} is not set.`,
+        `rerank=true requested but ${this.config.embeddings.apiKeyEnv} is not set.`,
         400
       );
     }
-    const candidates = ranked.map(({ hit }) => ({
-      id: hit.id,
-      text: [hit.metadata.title, hit.metadata.sectionTitle, hit.metadata.snippet].filter(Boolean).join("\n")
-    }));
+    const pageGroups = /* @__PURE__ */ new Map();
+    for (const entry of ranked) {
+      const url = entry.hit.metadata.url;
+      const group = pageGroups.get(url);
+      if (group) group.push(entry);
+      else pageGroups.set(url, [entry]);
+    }
+    const pageCandidates = [];
+    for (const [url, chunks] of pageGroups) {
+      const sorted = [...chunks].sort(
+        (a, b) => (a.hit.metadata.ordinal ?? 0) - (b.hit.metadata.ordinal ?? 0)
+      );
+      const title = sorted[0].hit.metadata.title;
+      const body = sorted.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
+      pageCandidates.push({ id: url, text: `${title}
+
+${body}` });
+    }
     const reranked = await this.reranker.rerank(
       query,
-      candidates,
+      pageCandidates,
       Math.max(topK, this.config.rerank.topN)
     );
-    const rerankScoreById = new Map(reranked.map((entry) => [entry.id, entry.score]));
+    const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {
-      const rerankScore = rerankScoreById.get(entry.hit.id);
-      const safeBaseScore = Number.isFinite(entry.finalScore) ? entry.finalScore : Number.NEGATIVE_INFINITY;
-      if (rerankScore === void 0 || !Number.isFinite(rerankScore)) {
-        return {
-          ...entry,
-          finalScore: safeBaseScore
-        };
+      const pageScore = scoreByUrl.get(entry.hit.metadata.url);
+      const base = Number.isFinite(entry.finalScore) ? entry.finalScore : Number.NEGATIVE_INFINITY;
+      if (pageScore === void 0 || !Number.isFinite(pageScore)) {
+        return { ...entry, finalScore: base };
       }
-      const combinedScore = rerankScore * this.config.ranking.weights.rerank + safeBaseScore * 1e-3;
+      const combined = pageScore * this.config.ranking.weights.rerank + base * 1e-3;
       return {
         ...entry,
-        finalScore: Number.isFinite(combinedScore) ? combinedScore : safeBaseScore
+        finalScore: Number.isFinite(combined) ? combined : base
       };
     }).sort((a, b) => {
       const delta = b.finalScore - a.finalScore;
@@ -18103,13 +18216,21 @@ function searchsocketHandle(options = {}) {
   let rateLimiter = null;
   const getConfig = async () => {
     if (!configPromise) {
-      const configP = options.config ? Promise.resolve(options.config) : loadConfig({
-        cwd: options.cwd,
-        configPath: options.configPath
-      });
+      let configP;
+      if (options.config) {
+        configP = Promise.resolve(options.config);
+      } else if (options.rawConfig) {
+        const cwd = options.cwd ?? process.cwd();
+        configP = Promise.resolve(mergeConfig(cwd, options.rawConfig));
+      } else {
+        configP = loadConfig({
+          cwd: options.cwd,
+          configPath: options.configPath
+        });
+      }
       configPromise = configP.then((config) => {
         apiPath = apiPath ?? config.api.path;
-        if (config.api.rateLimit) {
+        if (config.api.rateLimit && !isServerless()) {
           rateLimiter = new InMemoryRateLimiter(config.api.rateLimit.windowMs, config.api.rateLimit.max);
         }
         return config;
@@ -18119,10 +18240,9 @@ function searchsocketHandle(options = {}) {
   };
   const getEngine = async () => {
     if (!enginePromise) {
-      const config = options.config;
+      const config = await getConfig();
       enginePromise = SearchEngine.create({
         cwd: options.cwd,
-        configPath: options.configPath,
         config
       });
     }
@@ -19657,14 +19777,16 @@ function mapUrlToRoute(urlPath, patterns) {
 var Logger = class {
   json;
   verbose;
+  quiet;
   stderrOnly;
   constructor(opts = {}) {
     this.json = opts.json ?? false;
     this.verbose = opts.verbose ?? false;
+    this.quiet = opts.quiet ?? false;
     this.stderrOnly = opts.stderrOnly ?? false;
   }
   info(message) {
-    if (this.json) {
+    if (this.quiet || this.json) {
       return;
     }
     this.writeOut(`${message}
@@ -19678,7 +19800,7 @@ var Logger = class {
       this.logJson("debug", { message });
       return;
     }
-    this.writeOut(`${message}
+    this.writeOut(`  ${message}
 `);
   }
   warn(message) {
@@ -19705,7 +19827,7 @@ var Logger = class {
       this.logJson(event, data);
       return;
     }
-    this.writeOut(`[${event}] ${data ? JSON.stringify(data) : ""}
+    this.writeOut(`  [${event}] ${data ? JSON.stringify(data) : ""}
 `);
   }
   writeOut(text) {
@@ -19890,11 +20012,108 @@ async function startPreviewServer(cwd, options, logger3) {
 
 // src/indexing/sources/build/index.ts
 var logger = new Logger();
+function extractLinksFromHtml(html, pageUrl, baseOrigin) {
+  const $ = load(html);
+  const links = [];
+  $("a[href]").each((_i, el) => {
+    const href = $(el).attr("href");
+    if (!href || href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) {
+      return;
+    }
+    try {
+      const resolved = new URL(href, `${baseOrigin}${pageUrl}`);
+      if (resolved.origin !== baseOrigin) return;
+      if (!["http:", "https:"].includes(resolved.protocol)) return;
+      links.push(normalizeUrlPath(resolved.pathname));
+    } catch {
+    }
+  });
+  return [...new Set(links)];
+}
+async function discoverPages(server, buildConfig, pipelineMaxPages) {
+  const { seedUrls, maxDepth, exclude } = buildConfig;
+  const baseOrigin = new URL(server.baseUrl).origin;
+  let effectiveMax = buildConfig.maxPages;
+  if (typeof pipelineMaxPages === "number") {
+    const floored = Math.max(0, Math.floor(pipelineMaxPages));
+    effectiveMax = Math.min(effectiveMax, floored);
+  }
+  if (effectiveMax === 0) return [];
+  const visited = /* @__PURE__ */ new Set();
+  const pages = [];
+  const queue = [];
+  const limit = pLimit2(8);
+  for (const seed of seedUrls) {
+    const normalized = normalizeUrlPath(seed);
+    if (!visited.has(normalized) && !isExcluded(normalized, exclude)) {
+      visited.add(normalized);
+      queue.push({ url: normalized, depth: 0 });
+    }
+  }
+  while (queue.length > 0 && pages.length < effectiveMax) {
+    const remaining = effectiveMax - pages.length;
+    const batch = queue.splice(0, remaining);
+    const results = await Promise.allSettled(
+      batch.map(
+        (item) => limit(async () => {
+          const fullUrl = joinUrl(server.baseUrl, item.url);
+          const response = await fetch(fullUrl);
+          if (!response.ok) {
+            logger.warn(`Skipping ${item.url}: ${response.status} ${response.statusText}`);
+            return null;
+          }
+          const contentType = response.headers.get("content-type") ?? "";
+          if (!contentType.includes("text/html")) {
+            return null;
+          }
+          const html = await response.text();
+          if (item.depth < maxDepth) {
+            const links = extractLinksFromHtml(html, item.url, baseOrigin);
+            for (const link of links) {
+              if (!visited.has(link) && !isExcluded(link, exclude)) {
+                visited.add(link);
+                queue.push({ url: link, depth: item.depth + 1 });
+              }
+            }
+          }
+          return {
+            url: item.url,
+            html,
+            sourcePath: fullUrl,
+            outgoingLinks: []
+          };
+        })
+      )
+    );
+    for (const result of results) {
+      if (result.status === "fulfilled" && result.value) {
+        pages.push(result.value);
+      }
+    }
+  }
+  if (pages.length >= effectiveMax && queue.length > 0) {
+    logger.warn(`Discovery crawl reached maxPages limit (${effectiveMax}), ${queue.length} URLs not visited.`);
+  }
+  logger.event("build_discover_complete", {
+    pagesFound: pages.length,
+    urlsVisited: visited.size,
+    urlsSkipped: queue.length
+  });
+  return pages;
+}
 async function loadBuildPages(cwd, config, maxPages) {
   const buildConfig = config.source.build;
   if (!buildConfig) {
     throw new Error("build source config is missing");
   }
+  if (buildConfig.discover) {
+    const server2 = await startPreviewServer(cwd, { previewTimeout: buildConfig.previewTimeout }, logger);
+    try {
+      return await discoverPages(server2, buildConfig, maxPages);
+    } finally {
+      await server2.shutdown();
+    }
+  }
   const routes = await parseManifest(cwd, buildConfig.outputDir);
   const expanded = expandRoutes(routes, buildConfig.paramValues, buildConfig.exclude, logger);
   logger.event("build_routes_discovered", {
@@ -19905,7 +20124,7 @@ async function loadBuildPages(cwd, config, maxPages) {
   const selected = typeof maxCount === "number" ? expanded.slice(0, maxCount) : expanded;
   const server = await startPreviewServer(cwd, { previewTimeout: buildConfig.previewTimeout }, logger);
   try {
-    const concurrencyLimit = pLimit(8);
+    const concurrencyLimit = pLimit2(8);
     const results = await Promise.allSettled(
       selected.map(
         (route) => concurrencyLimit(async () => {
@@ -20074,7 +20293,7 @@ async function loadCrawledPages(config, maxPages) {
   const routes = await resolveRoutes(config);
   const maxCount = typeof maxPages === "number" ? Math.max(0, Math.floor(maxPages)) : void 0;
   const selected = typeof maxCount === "number" ? routes.slice(0, maxCount) : routes;
-  const concurrencyLimit = pLimit(8);
+  const concurrencyLimit = pLimit2(8);
   const results = await Promise.allSettled(
     selected.map(
       (route) => concurrencyLimit(async () => {
@@ -20128,9 +20347,7 @@ async function loadStaticOutputPages(cwd, config, maxPages) {
 
 // src/indexing/pipeline.ts
 var EMBEDDING_PRICE_PER_1K_TOKENS_USD = {
-  "text-embedding-3-small": 2e-5,
-  "text-embedding-3-large": 13e-5,
-  "text-embedding-ada-002": 1e-4
+  "jina-embeddings-v3": 2e-5
 };
 var DEFAULT_EMBEDDING_PRICE_PER_1K = 2e-5;
 var IndexPipeline = class _IndexPipeline {
@@ -20176,9 +20393,15 @@ var IndexPipeline = class _IndexPipeline {
     };
     const scope = resolveScope(this.config, options.scopeOverride);
     const { statePath } = ensureStateDirs(this.cwd, this.config.state.dir, scope);
+    const sourceMode = options.sourceOverride ?? this.config.source.mode;
+    this.logger.info(`Indexing scope "${scope.scopeName}" (source: ${sourceMode}, model: ${this.config.embeddings.model})`);
     if (options.force) {
+      this.logger.info("Force mode enabled \u2014 full rebuild");
       await cleanMirrorForScope(statePath, scope);
     }
+    if (options.dryRun) {
+      this.logger.info("Dry run \u2014 no writes will be performed");
+    }
     const manifestStart = stageStart();
     const existingHashes = await this.vectorStore.getContentHashes(scope);
     const existingModelId = await this.vectorStore.getScopeModelId(scope);
@@ -20189,8 +20412,9 @@ var IndexPipeline = class _IndexPipeline {
       );
     }
     stageEnd("manifest", manifestStart);
+    this.logger.debug(`Manifest: ${existingHashes.size} existing chunk hashes loaded`);
     const sourceStart = stageStart();
-    const sourceMode = options.sourceOverride ?? this.config.source.mode;
+    this.logger.info(`Loading pages (source: ${sourceMode})...`);
     let sourcePages;
     if (sourceMode === "static-output") {
       sourcePages = await loadStaticOutputPages(this.cwd, this.config, options.maxPages);
@@ -20202,10 +20426,13 @@ var IndexPipeline = class _IndexPipeline {
       sourcePages = await loadContentFilesPages(this.cwd, this.config, options.maxPages);
     }
     stageEnd("source", sourceStart);
+    this.logger.info(`Loaded ${sourcePages.length} page${sourcePages.length === 1 ? "" : "s"} (${stageTimingsMs["source"]}ms)`);
     const routeStart = stageStart();
     const routePatterns = await buildRoutePatterns(this.cwd);
     stageEnd("route_map", routeStart);
+    this.logger.debug(`Route mapping: ${routePatterns.length} pattern${routePatterns.length === 1 ? "" : "s"} discovered (${stageTimingsMs["route_map"]}ms)`);
     const extractStart = stageStart();
+    this.logger.info("Extracting content...");
     const extractedPages = [];
     for (const sourcePage of sourcePages) {
       const extracted = sourcePage.html ? extractFromHtml(sourcePage.url, sourcePage.html, this.config) : extractFromMarkdown(sourcePage.url, sourcePage.markdown ?? "", sourcePage.title);
@@ -20234,6 +20461,8 @@ var IndexPipeline = class _IndexPipeline {
       uniquePages.push(page);
     }
     stageEnd("extract", extractStart);
+    const skippedPages = sourcePages.length - uniquePages.length;
+    this.logger.info(`Extracted ${uniquePages.length} page${uniquePages.length === 1 ? "" : "s"}${skippedPages > 0 ? ` (${skippedPages} skipped)` : ""} (${stageTimingsMs["extract"]}ms)`);
     const linkStart = stageStart();
     const pageSet = new Set(uniquePages.map((page) => normalizeUrlPath(page.url)));
     const incomingLinkCount = /* @__PURE__ */ new Map();
@@ -20249,7 +20478,9 @@ var IndexPipeline = class _IndexPipeline {
       }
     }
     stageEnd("links", linkStart);
+    this.logger.debug(`Link analysis: computed incoming links for ${incomingLinkCount.size} pages (${stageTimingsMs["links"]}ms)`);
     const mirrorStart = stageStart();
+    this.logger.info("Writing mirror pages...");
     const mirrorPages = [];
     let routeExact = 0;
     let routeBestEffort = 0;
@@ -20319,7 +20550,9 @@ var IndexPipeline = class _IndexPipeline {
       await this.vectorStore.upsertPages(pageRecords, scope);
     }
     stageEnd("mirror", mirrorStart);
+    this.logger.info(`Mirrored ${mirrorPages.length} page${mirrorPages.length === 1 ? "" : "s"} (${routeExact} exact, ${routeBestEffort} best-effort) (${stageTimingsMs["mirror"]}ms)`);
     const chunkStart = stageStart();
+    this.logger.info("Chunking pages...");
     let chunks = mirrorPages.flatMap((page) => chunkMirrorPage(page, this.config, scope));
     const maxChunks = typeof options.maxChunks === "number" ? Math.max(0, Math.floor(options.maxChunks)) : void 0;
     if (typeof maxChunks === "number") {
@@ -20332,6 +20565,7 @@ var IndexPipeline = class _IndexPipeline {
       });
     }
     stageEnd("chunk", chunkStart);
+    this.logger.info(`Chunked into ${chunks.length} chunk${chunks.length === 1 ? "" : "s"} (${stageTimingsMs["chunk"]}ms)`);
     const currentChunkMap = /* @__PURE__ */ new Map();
     for (const chunk of chunks) {
       currentChunkMap.set(chunk.chunkKey, chunk);
@@ -20350,6 +20584,7 @@ var IndexPipeline = class _IndexPipeline {
       return existingHash !== chunk.contentHash;
     });
     const deletes = [...existingHashes.keys()].filter((chunkKey) => !currentChunkMap.has(chunkKey));
+    this.logger.info(`Changes detected: ${changedChunks.length} changed, ${deletes.length} deleted, ${chunks.length - changedChunks.length} unchanged`);
     const embedStart = stageStart();
     const chunkTokenEstimates = /* @__PURE__ */ new Map();
     for (const chunk of changedChunks) {
@@ -20364,9 +20599,11 @@ var IndexPipeline = class _IndexPipeline {
     let newEmbeddings = 0;
     const vectorsByChunk = /* @__PURE__ */ new Map();
     if (!options.dryRun && changedChunks.length > 0) {
+      this.logger.info(`Embedding ${changedChunks.length} chunk${changedChunks.length === 1 ? "" : "s"} (~${estimatedTokens.toLocaleString()} tokens, ~$${estimatedCostUSD.toFixed(6)})...`);
       const embeddings = await this.embeddings.embedTexts(
         changedChunks.map((chunk) => buildEmbeddingText(chunk, this.config.chunking.prependTitle)),
-        this.config.embeddings.model
+        this.config.embeddings.model,
+        "retrieval.passage"
       );
       if (embeddings.length !== changedChunks.length) {
         throw new SearchSocketError(
@@ -20389,8 +20626,14 @@ var IndexPipeline = class _IndexPipeline {
       }
     }
     stageEnd("embedding", embedStart);
+    if (changedChunks.length > 0) {
+      this.logger.info(`Embedded ${newEmbeddings} chunk${newEmbeddings === 1 ? "" : "s"} (${stageTimingsMs["embedding"]}ms)`);
+    } else {
+      this.logger.info("No chunks to embed \u2014 all up to date");
+    }
     const syncStart = stageStart();
     if (!options.dryRun) {
+      this.logger.info("Syncing vectors...");
       const upserts = [];
       for (const chunk of changedChunks) {
         const vector = vectorsByChunk.get(chunk.chunkKey);
@@ -20409,6 +20652,8 @@ var IndexPipeline = class _IndexPipeline {
             sectionTitle: chunk.sectionTitle ?? "",
             headingPath: chunk.headingPath,
             snippet: chunk.snippet,
+            chunkText: chunk.chunkText.slice(0, 4e3),
+            ordinal: chunk.ordinal,
             contentHash: chunk.contentHash,
             modelId: this.config.embeddings.model,
             depth: chunk.depth,
@@ -20428,6 +20673,7 @@ var IndexPipeline = class _IndexPipeline {
       }
     }
     stageEnd("sync", syncStart);
+    this.logger.debug(`Sync complete (${stageTimingsMs["sync"]}ms)`);
     const finalizeStart = stageStart();
     if (!options.dryRun) {
       const scopeInfo = {
@@ -20447,6 +20693,7 @@ var IndexPipeline = class _IndexPipeline {
       });
     }
     stageEnd("finalize", finalizeStart);
+    this.logger.info("Done.");
     return {
       pagesProcessed: mirrorPages.length,
       chunksTotal: chunks.length,