searchsocket 0.3.1 → 0.3.3

package/dist/cli.js

@@ -12,7 +12,7 @@ import { Command } from "commander";
 // package.json
 var package_default = {
   name: "searchsocket",
-  version: "0.3.1",
+  version: "0.3.3",
   description: "Semantic site search and MCP retrieval for SvelteKit static sites",
   license: "MIT",
   author: "Greg Priday <greg@siteorigin.com>",
@@ -189,7 +189,7 @@ var searchSocketConfigSchema = z.object({
   ranking: z.object({
     enableIncomingLinkBoost: z.boolean().optional(),
     enableDepthBoost: z.boolean().optional(),
-    pageWeights: z.record(z.string(), z.number().positive()).optional(),
+    pageWeights: z.record(z.string(), z.number().min(0)).optional(),
     aggregationCap: z.number().int().positive().optional(),
     aggregationDecay: z.number().min(0).max(1).optional(),
     minChunkScoreRatio: z.number().min(0).max(1).optional(),
@@ -1700,7 +1700,7 @@ function extractFromHtml(url, html, config) {
   const $ = load(html);
   const normalizedUrl = normalizeUrlPath(url);
   const pageBaseUrl = new URL(`https://searchsocket.local${normalizedUrl}`);
-  const title = normalizeText($("title").first().text() || "") || normalizeText($(`${config.extract.mainSelector} h1`).first().text() || "") || normalizedUrl;
+  const title = $("meta[property='og:title']").attr("content")?.trim() || normalizeText($(`${config.extract.mainSelector} h1`).first().text() || "") || $("meta[name='twitter:title']").attr("content")?.trim() || normalizeText($("title").first().text() || "") || normalizedUrl;
   if (config.extract.respectRobotsNoindex) {
     const robots = $("meta[name='robots']").attr("content") ?? "";
     if (/\bnoindex\b/i.test(robots)) {
@@ -2854,7 +2854,7 @@ var JinaReranker = class {
   constructor(options) {
     this.apiKey = options.apiKey;
     this.model = options.model;
-    this.maxRetries = options.maxRetries ?? 4;
+    this.maxRetries = options.maxRetries ?? 2;
   }
   async rerank(query, candidates, topN) {
     if (candidates.length === 0) {
@@ -2864,7 +2864,8 @@ var JinaReranker = class {
       model: this.model,
       query,
       documents: candidates.map((candidate) => candidate.text),
-      top_n: topN ?? candidates.length
+      top_n: topN ?? candidates.length,
+      return_documents: false
     };
     let attempt = 0;
     while (attempt <= this.maxRetries) {
@@ -3010,6 +3011,7 @@ function aggregateByPage(ranked, config) {
     }
     let pageScore = maxScore + aggregationBonus * config.ranking.weights.aggregation;
     const pageWeight = findPageWeight(url, config.ranking.pageWeights);
+    if (pageWeight === 0) continue;
     if (pageWeight !== 1) {
       pageScore *= pageWeight;
     }
@@ -3235,6 +3237,7 @@ var SearchEngine = class _SearchEngine {
     const MAX_CHUNKS_PER_PAGE = 5;
     const MIN_CHUNKS_PER_PAGE = 1;
     const MIN_CHUNK_SCORE_RATIO = 0.5;
+    const MAX_DOC_CHARS = 2e3;
     const pageCandidates = [];
     for (const [url, chunks] of pageGroups) {
       const byScore = [...chunks].sort((a, b) => b.finalScore - a.finalScore);
@@ -3254,12 +3257,18 @@ var SearchEngine = class _SearchEngine {
       }
       const body = selected.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
       parts.push(body);
-      pageCandidates.push({ id: url, text: parts.join("\n\n") });
+      let text = parts.join("\n\n");
+      if (text.length > MAX_DOC_CHARS) {
+        text = text.slice(0, MAX_DOC_CHARS);
+      }
+      pageCandidates.push({ id: url, text });
     }
+    const maxCandidates = Math.max(topK, this.config.rerank.topN);
+    const cappedCandidates = pageCandidates.slice(0, maxCandidates);
     const reranked = await this.reranker.rerank(
       query,
-      pageCandidates,
-      Math.max(topK, this.config.rerank.topN)
+      cappedCandidates,
+      maxCandidates
     );
     const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {

package/dist/index.cjs

@@ -16688,7 +16688,7 @@ var searchSocketConfigSchema = zod.z.object({
   ranking: zod.z.object({
     enableIncomingLinkBoost: zod.z.boolean().optional(),
     enableDepthBoost: zod.z.boolean().optional(),
-    pageWeights: zod.z.record(zod.z.string(), zod.z.number().positive()).optional(),
+    pageWeights: zod.z.record(zod.z.string(), zod.z.number().min(0)).optional(),
     aggregationCap: zod.z.number().int().positive().optional(),
     aggregationDecay: zod.z.number().min(0).max(1).optional(),
     minChunkScoreRatio: zod.z.number().min(0).max(1).optional(),
@@ -17273,7 +17273,7 @@ var JinaReranker = class {
   constructor(options) {
     this.apiKey = options.apiKey;
     this.model = options.model;
-    this.maxRetries = options.maxRetries ?? 4;
+    this.maxRetries = options.maxRetries ?? 2;
   }
   async rerank(query, candidates, topN) {
     if (candidates.length === 0) {
@@ -17283,7 +17283,8 @@ var JinaReranker = class {
       model: this.model,
       query,
       documents: candidates.map((candidate) => candidate.text),
-      top_n: topN ?? candidates.length
+      top_n: topN ?? candidates.length,
+      return_documents: false
     };
     let attempt = 0;
     while (attempt <= this.maxRetries) {
@@ -19038,7 +19039,7 @@ function extractFromHtml(url, html, config) {
   const $ = cheerio.load(html);
   const normalizedUrl = normalizeUrlPath(url);
   const pageBaseUrl = new URL(`https://searchsocket.local${normalizedUrl}`);
-  const title = normalizeText($("title").first().text() || "") || normalizeText($(`${config.extract.mainSelector} h1`).first().text() || "") || normalizedUrl;
+  const title = $("meta[property='og:title']").attr("content")?.trim() || normalizeText($(`${config.extract.mainSelector} h1`).first().text() || "") || $("meta[name='twitter:title']").attr("content")?.trim() || normalizeText($("title").first().text() || "") || normalizedUrl;
   if (config.extract.respectRobotsNoindex) {
     const robots = $("meta[name='robots']").attr("content") ?? "";
     if (/\bnoindex\b/i.test(robots)) {
@@ -20279,6 +20280,7 @@ function aggregateByPage(ranked, config) {
     }
     let pageScore = maxScore + aggregationBonus * config.ranking.weights.aggregation;
     const pageWeight = findPageWeight(url, config.ranking.pageWeights);
+    if (pageWeight === 0) continue;
     if (pageWeight !== 1) {
       pageScore *= pageWeight;
     }
@@ -20504,6 +20506,7 @@ var SearchEngine = class _SearchEngine {
     const MAX_CHUNKS_PER_PAGE = 5;
     const MIN_CHUNKS_PER_PAGE = 1;
     const MIN_CHUNK_SCORE_RATIO = 0.5;
+    const MAX_DOC_CHARS = 2e3;
     const pageCandidates = [];
     for (const [url, chunks] of pageGroups) {
       const byScore = [...chunks].sort((a, b) => b.finalScore - a.finalScore);
@@ -20523,12 +20526,18 @@ var SearchEngine = class _SearchEngine {
       }
       const body = selected.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
       parts.push(body);
-      pageCandidates.push({ id: url, text: parts.join("\n\n") });
+      let text = parts.join("\n\n");
+      if (text.length > MAX_DOC_CHARS) {
+        text = text.slice(0, MAX_DOC_CHARS);
+      }
+      pageCandidates.push({ id: url, text });
     }
+    const maxCandidates = Math.max(topK, this.config.rerank.topN);
+    const cappedCandidates = pageCandidates.slice(0, maxCandidates);
     const reranked = await this.reranker.rerank(
       query,
-      pageCandidates,
-      Math.max(topK, this.config.rerank.topN)
+      cappedCandidates,
+      maxCandidates
     );
     const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {

package/dist/index.js

@@ -16676,7 +16676,7 @@ var searchSocketConfigSchema = z.object({
   ranking: z.object({
     enableIncomingLinkBoost: z.boolean().optional(),
     enableDepthBoost: z.boolean().optional(),
-    pageWeights: z.record(z.string(), z.number().positive()).optional(),
+    pageWeights: z.record(z.string(), z.number().min(0)).optional(),
     aggregationCap: z.number().int().positive().optional(),
     aggregationDecay: z.number().min(0).max(1).optional(),
     minChunkScoreRatio: z.number().min(0).max(1).optional(),
@@ -17261,7 +17261,7 @@ var JinaReranker = class {
   constructor(options) {
     this.apiKey = options.apiKey;
     this.model = options.model;
-    this.maxRetries = options.maxRetries ?? 4;
+    this.maxRetries = options.maxRetries ?? 2;
   }
   async rerank(query, candidates, topN) {
     if (candidates.length === 0) {
@@ -17271,7 +17271,8 @@ var JinaReranker = class {
       model: this.model,
       query,
       documents: candidates.map((candidate) => candidate.text),
-      top_n: topN ?? candidates.length
+      top_n: topN ?? candidates.length,
+      return_documents: false
     };
     let attempt = 0;
     while (attempt <= this.maxRetries) {
@@ -19026,7 +19027,7 @@ function extractFromHtml(url, html, config) {
   const $ = load(html);
   const normalizedUrl = normalizeUrlPath(url);
   const pageBaseUrl = new URL(`https://searchsocket.local${normalizedUrl}`);
-  const title = normalizeText($("title").first().text() || "") || normalizeText($(`${config.extract.mainSelector} h1`).first().text() || "") || normalizedUrl;
+  const title = $("meta[property='og:title']").attr("content")?.trim() || normalizeText($(`${config.extract.mainSelector} h1`).first().text() || "") || $("meta[name='twitter:title']").attr("content")?.trim() || normalizeText($("title").first().text() || "") || normalizedUrl;
   if (config.extract.respectRobotsNoindex) {
     const robots = $("meta[name='robots']").attr("content") ?? "";
     if (/\bnoindex\b/i.test(robots)) {
@@ -20267,6 +20268,7 @@ function aggregateByPage(ranked, config) {
     }
     let pageScore = maxScore + aggregationBonus * config.ranking.weights.aggregation;
     const pageWeight = findPageWeight(url, config.ranking.pageWeights);
+    if (pageWeight === 0) continue;
     if (pageWeight !== 1) {
       pageScore *= pageWeight;
     }
@@ -20492,6 +20494,7 @@ var SearchEngine = class _SearchEngine {
     const MAX_CHUNKS_PER_PAGE = 5;
     const MIN_CHUNKS_PER_PAGE = 1;
     const MIN_CHUNK_SCORE_RATIO = 0.5;
+    const MAX_DOC_CHARS = 2e3;
     const pageCandidates = [];
     for (const [url, chunks] of pageGroups) {
       const byScore = [...chunks].sort((a, b) => b.finalScore - a.finalScore);
@@ -20511,12 +20514,18 @@ var SearchEngine = class _SearchEngine {
       }
       const body = selected.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
       parts.push(body);
-      pageCandidates.push({ id: url, text: parts.join("\n\n") });
+      let text = parts.join("\n\n");
+      if (text.length > MAX_DOC_CHARS) {
+        text = text.slice(0, MAX_DOC_CHARS);
+      }
+      pageCandidates.push({ id: url, text });
     }
+    const maxCandidates = Math.max(topK, this.config.rerank.topN);
+    const cappedCandidates = pageCandidates.slice(0, maxCandidates);
     const reranked = await this.reranker.rerank(
       query,
-      pageCandidates,
-      Math.max(topK, this.config.rerank.topN)
+      cappedCandidates,
+      maxCandidates
     );
     const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {

package/dist/sveltekit.cjs

@@ -16684,7 +16684,7 @@ var searchSocketConfigSchema = zod.z.object({
   ranking: zod.z.object({
     enableIncomingLinkBoost: zod.z.boolean().optional(),
     enableDepthBoost: zod.z.boolean().optional(),
-    pageWeights: zod.z.record(zod.z.string(), zod.z.number().positive()).optional(),
+    pageWeights: zod.z.record(zod.z.string(), zod.z.number().min(0)).optional(),
     aggregationCap: zod.z.number().int().positive().optional(),
     aggregationDecay: zod.z.number().min(0).max(1).optional(),
     minChunkScoreRatio: zod.z.number().min(0).max(1).optional(),
@@ -17254,7 +17254,7 @@ var JinaReranker = class {
   constructor(options) {
     this.apiKey = options.apiKey;
     this.model = options.model;
-    this.maxRetries = options.maxRetries ?? 4;
+    this.maxRetries = options.maxRetries ?? 2;
   }
   async rerank(query, candidates, topN) {
     if (candidates.length === 0) {
@@ -17264,7 +17264,8 @@ var JinaReranker = class {
       model: this.model,
       query,
       documents: candidates.map((candidate) => candidate.text),
-      top_n: topN ?? candidates.length
+      top_n: topN ?? candidates.length,
+      return_documents: false
     };
     let attempt = 0;
     while (attempt <= this.maxRetries) {
@@ -17940,6 +17941,7 @@ function aggregateByPage(ranked, config) {
     }
     let pageScore = maxScore + aggregationBonus * config.ranking.weights.aggregation;
     const pageWeight = findPageWeight(url, config.ranking.pageWeights);
+    if (pageWeight === 0) continue;
     if (pageWeight !== 1) {
       pageScore *= pageWeight;
     }
@@ -18165,6 +18167,7 @@ var SearchEngine = class _SearchEngine {
     const MAX_CHUNKS_PER_PAGE = 5;
     const MIN_CHUNKS_PER_PAGE = 1;
     const MIN_CHUNK_SCORE_RATIO = 0.5;
+    const MAX_DOC_CHARS = 2e3;
     const pageCandidates = [];
     for (const [url, chunks] of pageGroups) {
       const byScore = [...chunks].sort((a, b) => b.finalScore - a.finalScore);
@@ -18184,12 +18187,18 @@ var SearchEngine = class _SearchEngine {
       }
       const body = selected.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
       parts.push(body);
-      pageCandidates.push({ id: url, text: parts.join("\n\n") });
+      let text = parts.join("\n\n");
+      if (text.length > MAX_DOC_CHARS) {
+        text = text.slice(0, MAX_DOC_CHARS);
+      }
+      pageCandidates.push({ id: url, text });
     }
+    const maxCandidates = Math.max(topK, this.config.rerank.topN);
+    const cappedCandidates = pageCandidates.slice(0, maxCandidates);
     const reranked = await this.reranker.rerank(
       query,
-      pageCandidates,
-      Math.max(topK, this.config.rerank.topN)
+      cappedCandidates,
+      maxCandidates
     );
     const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {
@@ -19580,7 +19589,7 @@ function extractFromHtml(url, html, config) {
   const $ = cheerio.load(html);
   const normalizedUrl = normalizeUrlPath(url);
   const pageBaseUrl = new URL(`https://searchsocket.local${normalizedUrl}`);
-  const title = normalizeText($("title").first().text() || "") || normalizeText($(`${config.extract.mainSelector} h1`).first().text() || "") || normalizedUrl;
+  const title = $("meta[property='og:title']").attr("content")?.trim() || normalizeText($(`${config.extract.mainSelector} h1`).first().text() || "") || $("meta[name='twitter:title']").attr("content")?.trim() || normalizeText($("title").first().text() || "") || normalizedUrl;
   if (config.extract.respectRobotsNoindex) {
     const robots = $("meta[name='robots']").attr("content") ?? "";
     if (/\bnoindex\b/i.test(robots)) {

package/dist/sveltekit.js

@@ -16672,7 +16672,7 @@ var searchSocketConfigSchema = z.object({
   ranking: z.object({
     enableIncomingLinkBoost: z.boolean().optional(),
     enableDepthBoost: z.boolean().optional(),
-    pageWeights: z.record(z.string(), z.number().positive()).optional(),
+    pageWeights: z.record(z.string(), z.number().min(0)).optional(),
     aggregationCap: z.number().int().positive().optional(),
     aggregationDecay: z.number().min(0).max(1).optional(),
     minChunkScoreRatio: z.number().min(0).max(1).optional(),
@@ -17242,7 +17242,7 @@ var JinaReranker = class {
   constructor(options) {
     this.apiKey = options.apiKey;
     this.model = options.model;
-    this.maxRetries = options.maxRetries ?? 4;
+    this.maxRetries = options.maxRetries ?? 2;
   }
   async rerank(query, candidates, topN) {
     if (candidates.length === 0) {
@@ -17252,7 +17252,8 @@ var JinaReranker = class {
       model: this.model,
       query,
       documents: candidates.map((candidate) => candidate.text),
-      top_n: topN ?? candidates.length
+      top_n: topN ?? candidates.length,
+      return_documents: false
     };
     let attempt = 0;
     while (attempt <= this.maxRetries) {
@@ -17928,6 +17929,7 @@ function aggregateByPage(ranked, config) {
     }
     let pageScore = maxScore + aggregationBonus * config.ranking.weights.aggregation;
     const pageWeight = findPageWeight(url, config.ranking.pageWeights);
+    if (pageWeight === 0) continue;
     if (pageWeight !== 1) {
       pageScore *= pageWeight;
     }
@@ -18153,6 +18155,7 @@ var SearchEngine = class _SearchEngine {
     const MAX_CHUNKS_PER_PAGE = 5;
     const MIN_CHUNKS_PER_PAGE = 1;
     const MIN_CHUNK_SCORE_RATIO = 0.5;
+    const MAX_DOC_CHARS = 2e3;
     const pageCandidates = [];
     for (const [url, chunks] of pageGroups) {
       const byScore = [...chunks].sort((a, b) => b.finalScore - a.finalScore);
@@ -18172,12 +18175,18 @@ var SearchEngine = class _SearchEngine {
       }
       const body = selected.map((c) => c.hit.metadata.chunkText || c.hit.metadata.snippet).join("\n\n");
       parts.push(body);
-      pageCandidates.push({ id: url, text: parts.join("\n\n") });
+      let text = parts.join("\n\n");
+      if (text.length > MAX_DOC_CHARS) {
+        text = text.slice(0, MAX_DOC_CHARS);
+      }
+      pageCandidates.push({ id: url, text });
     }
+    const maxCandidates = Math.max(topK, this.config.rerank.topN);
+    const cappedCandidates = pageCandidates.slice(0, maxCandidates);
     const reranked = await this.reranker.rerank(
       query,
-      pageCandidates,
-      Math.max(topK, this.config.rerank.topN)
+      cappedCandidates,
+      maxCandidates
     );
     const scoreByUrl = new Map(reranked.map((e) => [e.id, e.score]));
     return ranked.map((entry) => {
@@ -19568,7 +19577,7 @@ function extractFromHtml(url, html, config) {
   const $ = load(html);
   const normalizedUrl = normalizeUrlPath(url);
   const pageBaseUrl = new URL(`https://searchsocket.local${normalizedUrl}`);
-  const title = normalizeText($("title").first().text() || "") || normalizeText($(`${config.extract.mainSelector} h1`).first().text() || "") || normalizedUrl;
+  const title = $("meta[property='og:title']").attr("content")?.trim() || normalizeText($(`${config.extract.mainSelector} h1`).first().text() || "") || $("meta[name='twitter:title']").attr("content")?.trim() || normalizeText($("title").first().text() || "") || normalizedUrl;
   if (config.extract.respectRobotsNoindex) {
     const robots = $("meta[name='robots']").attr("content") ?? "";
     if (/\bnoindex\b/i.test(robots)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "searchsocket",
-  "version": "0.3.1",
+  "version": "0.3.3",
   "description": "Semantic site search and MCP retrieval for SvelteKit static sites",
   "license": "MIT",
   "author": "Greg Priday <greg@siteorigin.com>",