searchsocket 0.3.3 → 0.5.0

package/dist/{types-BrG6XTUU.d.cts → types-Dk43uz25.d.cts}

@@ -1,6 +1,5 @@
 type ScopeMode = "fixed" | "git" | "env";
 type SourceMode = "static-output" | "crawl" | "content-files" | "build";
-type EmbeddingProvider = "jina";
 interface SearchSocketConfig {
     project?: {
         id?: string;
@@ -12,6 +11,8 @@ interface SearchSocketConfig {
         envVar?: string;
         sanitize?: boolean;
     };
+    exclude?: string[];
+    respectRobotsTxt?: boolean;
     source?: {
         mode?: SourceMode;
         staticOutputDir?: string;
@@ -59,29 +60,18 @@ interface SearchSocketConfig {
         prependTitle?: boolean;
         pageSummaryChunk?: boolean;
     };
-    embeddings?: {
-        provider?: EmbeddingProvider;
-        model?: string;
-        apiKey?: string;
-        apiKeyEnv?: string;
-        batchSize?: number;
-        concurrency?: number;
-        pricePer1kTokens?: number;
+    upstash?: {
+        url?: string;
+        token?: string;
+        urlEnv?: string;
+        tokenEnv?: string;
     };
-    vector?: {
-        dimension?: number;
-        turso?: {
-            url?: string;
-            authToken?: string;
-            urlEnv?: string;
-            authTokenEnv?: string;
-            localPath?: string;
-        };
-    };
-    rerank?: {
-        enabled?: boolean;
-        topN?: number;
-        model?: string;
+    search?: {
+        semanticWeight?: number;
+        inputEnrichment?: boolean;
+        reranking?: boolean;
+        dualSearch?: boolean;
+        pageSearchWeight?: number;
     };
     ranking?: {
         enableIncomingLinkBoost?: boolean;
@@ -91,11 +81,12 @@ interface SearchSocketConfig {
         aggregationDecay?: number;
         minChunkScoreRatio?: number;
         minScore?: number;
+        scoreGapThreshold?: number;
         weights?: {
             incomingLinks?: number;
             depth?: number;
-            rerank?: number;
             aggregation?: number;
+            titleMatch?: number;
         };
     };
     api?: {
@@ -118,7 +109,6 @@ interface SearchSocketConfig {
     };
     state?: {
         dir?: string;
-        writeMirror?: boolean;
     };
 }
 interface ResolvedSearchSocketConfig {
@@ -132,6 +122,8 @@ interface ResolvedSearchSocketConfig {
         envVar: string;
         sanitize: boolean;
     };
+    exclude: string[];
+    respectRobotsTxt: boolean;
     source: {
         mode: SourceMode;
         staticOutputDir: string;
@@ -179,29 +171,18 @@ interface ResolvedSearchSocketConfig {
         prependTitle: boolean;
         pageSummaryChunk: boolean;
     };
-    embeddings: {
-        provider: EmbeddingProvider;
-        model: string;
-        apiKey?: string;
-        apiKeyEnv: string;
-        batchSize: number;
-        concurrency: number;
-        pricePer1kTokens?: number;
-    };
-    vector: {
-        dimension?: number;
-        turso: {
-            url?: string;
-            authToken?: string;
-            urlEnv: string;
-            authTokenEnv: string;
-            localPath: string;
-        };
+    upstash: {
+        url?: string;
+        token?: string;
+        urlEnv: string;
+        tokenEnv: string;
     };
-    rerank: {
-        enabled: boolean;
-        topN: number;
-        model: string;
+    search: {
+        semanticWeight: number;
+        inputEnrichment: boolean;
+        reranking: boolean;
+        dualSearch: boolean;
+        pageSearchWeight: number;
     };
     ranking: {
         enableIncomingLinkBoost: boolean;
@@ -211,11 +192,12 @@ interface ResolvedSearchSocketConfig {
         aggregationDecay: number;
         minChunkScoreRatio: number;
         minScore: number;
+        scoreGapThreshold: number;
         weights: {
             incomingLinks: number;
             depth: number;
-            rerank: number;
             aggregation: number;
+            titleMatch: number;
         };
     };
     api: {
@@ -238,7 +220,6 @@ interface ResolvedSearchSocketConfig {
     };
     state: {
         dir: string;
-        writeMirror: boolean;
     };
 }
 interface Scope {
@@ -264,9 +245,9 @@ interface Chunk {
     description?: string;
     keywords?: string[];
 }
-interface VectorRecord {
+interface VectorHit {
     id: string;
-    vector: number[];
+    score: number;
     metadata: {
         projectId: string;
         scopeName: string;
@@ -279,7 +260,6 @@ interface VectorRecord {
         chunkText: string;
         ordinal: number;
         contentHash: string;
-        modelId: string;
         depth: number;
         incomingLinks: number;
         routeFile: string;
@@ -288,16 +268,6 @@ interface VectorRecord {
         keywords?: string[];
     };
 }
-interface QueryOpts {
-    topK: number;
-    pathPrefix?: string;
-    tags?: string[];
-}
-interface VectorHit {
-    id: string;
-    score: number;
-    metadata: VectorRecord["metadata"];
-}
 interface PageRecord {
     url: string;
     title: string;
@@ -311,48 +281,26 @@ interface PageRecord {
     depth: number;
     tags: string[];
     indexedAt: string;
+    summary?: string;
+    description?: string;
+    keywords?: string[];
+}
+interface PageHit {
+    id: string;
+    score: number;
+    title: string;
+    url: string;
+    description: string;
+    tags: string[];
+    depth: number;
+    incomingLinks: number;
+    routeFile: string;
 }
 interface ScopeInfo {
     projectId: string;
     scopeName: string;
-    modelId: string;
     lastIndexedAt: string;
-    vectorCount?: number;
-    lastEstimateTokens?: number;
-    lastEstimateCostUSD?: number;
-    lastEstimateChangedChunks?: number;
-}
-interface VectorStore {
-    upsert(records: VectorRecord[], scope: Scope): Promise<void>;
-    query(queryVector: number[], opts: QueryOpts, scope: Scope): Promise<VectorHit[]>;
-    deleteByIds(ids: string[], scope: Scope): Promise<void>;
-    deleteScope(scope: Scope): Promise<void>;
-    listScopes(scopeProjectId: string): Promise<ScopeInfo[]>;
-    recordScope(info: ScopeInfo): Promise<void>;
-    health(): Promise<{
-        ok: boolean;
-        details?: string;
-    }>;
-    getContentHashes(scope: Scope): Promise<Map<string, string>>;
-    upsertPages(pages: PageRecord[], scope: Scope): Promise<void>;
-    getPage(url: string, scope: Scope): Promise<PageRecord | null>;
-    deletePages(scope: Scope): Promise<void>;
-    getScopeModelId(scope: Scope): Promise<string | null>;
-    dropAllTables(): Promise<void>;
-}
-interface EmbeddingsProvider {
-    embedTexts(texts: string[], modelId: string, task?: string): Promise<number[][]>;
-    estimateTokens(text: string): number;
-}
-interface RerankCandidate {
-    id: string;
-    text: string;
-}
-interface Reranker {
-    rerank(query: string, candidates: RerankCandidate[], topN?: number): Promise<Array<{
-        id: string;
-        score: number;
-    }>>;
+    documentCount?: number;
 }
 interface SearchRequest {
     q: string;
@@ -360,7 +308,6 @@ interface SearchRequest {
     scope?: string;
     pathPrefix?: string;
     tags?: string[];
-    rerank?: boolean;
     groupBy?: "page" | "chunk";
 }
 interface SearchResultChunk {
@@ -384,23 +331,17 @@ interface SearchResponse {
     results: SearchResult[];
     meta: {
         timingsMs: {
-            embed: number;
-            vector: number;
-            rerank: number;
+            search: number;
             total: number;
         };
-        usedRerank: boolean;
-        modelId: string;
     };
 }
 interface IndexStats {
     pagesProcessed: number;
     chunksTotal: number;
     chunksChanged: number;
-    newEmbeddings: number;
+    documentsUpserted: number;
     deletes: number;
-    estimatedTokens: number;
-    estimatedCostUSD: number;
     routeExact: number;
     routeBestEffort: number;
     stageTimingsMs: Record<string, number>;
@@ -416,4 +357,4 @@ interface IndexOptions {
     verbose?: boolean;
 }
 
-export type { Chunk as C, EmbeddingsProvider as E, IndexOptions as I, QueryOpts as Q, ResolvedSearchSocketConfig as R, SearchRequest as S, VectorStore as V, SearchResponse as a, SearchSocketConfig as b, Scope as c, Reranker as d, RerankCandidate as e, IndexStats as f, VectorHit as g, VectorRecord as h };
+export type { Chunk as C, IndexOptions as I, PageHit as P, ResolvedSearchSocketConfig as R, SearchRequest as S, VectorHit as V, SearchResponse as a, SearchResult as b, SearchSocketConfig as c, Scope as d, ScopeInfo as e, PageRecord as f, IndexStats as g };

package/dist/{types-BrG6XTUU.d.ts → types-Dk43uz25.d.ts}

@@ -1,6 +1,5 @@
 type ScopeMode = "fixed" | "git" | "env";
 type SourceMode = "static-output" | "crawl" | "content-files" | "build";
-type EmbeddingProvider = "jina";
 interface SearchSocketConfig {
     project?: {
         id?: string;
@@ -12,6 +11,8 @@ interface SearchSocketConfig {
         envVar?: string;
         sanitize?: boolean;
     };
+    exclude?: string[];
+    respectRobotsTxt?: boolean;
     source?: {
         mode?: SourceMode;
         staticOutputDir?: string;
@@ -59,29 +60,18 @@ interface SearchSocketConfig {
         prependTitle?: boolean;
         pageSummaryChunk?: boolean;
     };
-    embeddings?: {
-        provider?: EmbeddingProvider;
-        model?: string;
-        apiKey?: string;
-        apiKeyEnv?: string;
-        batchSize?: number;
-        concurrency?: number;
-        pricePer1kTokens?: number;
+    upstash?: {
+        url?: string;
+        token?: string;
+        urlEnv?: string;
+        tokenEnv?: string;
     };
-    vector?: {
-        dimension?: number;
-        turso?: {
-            url?: string;
-            authToken?: string;
-            urlEnv?: string;
-            authTokenEnv?: string;
-            localPath?: string;
-        };
-    };
-    rerank?: {
-        enabled?: boolean;
-        topN?: number;
-        model?: string;
+    search?: {
+        semanticWeight?: number;
+        inputEnrichment?: boolean;
+        reranking?: boolean;
+        dualSearch?: boolean;
+        pageSearchWeight?: number;
     };
     ranking?: {
         enableIncomingLinkBoost?: boolean;
@@ -91,11 +81,12 @@ interface SearchSocketConfig {
         aggregationDecay?: number;
         minChunkScoreRatio?: number;
         minScore?: number;
+        scoreGapThreshold?: number;
         weights?: {
             incomingLinks?: number;
             depth?: number;
-            rerank?: number;
             aggregation?: number;
+            titleMatch?: number;
         };
     };
     api?: {
@@ -118,7 +109,6 @@ interface SearchSocketConfig {
     };
     state?: {
         dir?: string;
-        writeMirror?: boolean;
     };
 }
 interface ResolvedSearchSocketConfig {
@@ -132,6 +122,8 @@ interface ResolvedSearchSocketConfig {
         envVar: string;
         sanitize: boolean;
     };
+    exclude: string[];
+    respectRobotsTxt: boolean;
     source: {
         mode: SourceMode;
         staticOutputDir: string;
@@ -179,29 +171,18 @@ interface ResolvedSearchSocketConfig {
         prependTitle: boolean;
         pageSummaryChunk: boolean;
     };
-    embeddings: {
-        provider: EmbeddingProvider;
-        model: string;
-        apiKey?: string;
-        apiKeyEnv: string;
-        batchSize: number;
-        concurrency: number;
-        pricePer1kTokens?: number;
-    };
-    vector: {
-        dimension?: number;
-        turso: {
-            url?: string;
-            authToken?: string;
-            urlEnv: string;
-            authTokenEnv: string;
-            localPath: string;
-        };
+    upstash: {
+        url?: string;
+        token?: string;
+        urlEnv: string;
+        tokenEnv: string;
     };
-    rerank: {
-        enabled: boolean;
-        topN: number;
-        model: string;
+    search: {
+        semanticWeight: number;
+        inputEnrichment: boolean;
+        reranking: boolean;
+        dualSearch: boolean;
+        pageSearchWeight: number;
     };
     ranking: {
         enableIncomingLinkBoost: boolean;
@@ -211,11 +192,12 @@ interface ResolvedSearchSocketConfig {
         aggregationDecay: number;
         minChunkScoreRatio: number;
         minScore: number;
+        scoreGapThreshold: number;
         weights: {
             incomingLinks: number;
             depth: number;
-            rerank: number;
             aggregation: number;
+            titleMatch: number;
         };
     };
     api: {
@@ -238,7 +220,6 @@ interface ResolvedSearchSocketConfig {
     };
     state: {
         dir: string;
-        writeMirror: boolean;
     };
 }
 interface Scope {
@@ -264,9 +245,9 @@ interface Chunk {
     description?: string;
     keywords?: string[];
 }
-interface VectorRecord {
+interface VectorHit {
     id: string;
-    vector: number[];
+    score: number;
     metadata: {
         projectId: string;
         scopeName: string;
@@ -279,7 +260,6 @@ interface VectorRecord {
         chunkText: string;
         ordinal: number;
         contentHash: string;
-        modelId: string;
         depth: number;
         incomingLinks: number;
         routeFile: string;
@@ -288,16 +268,6 @@ interface VectorRecord {
         keywords?: string[];
     };
 }
-interface QueryOpts {
-    topK: number;
-    pathPrefix?: string;
-    tags?: string[];
-}
-interface VectorHit {
-    id: string;
-    score: number;
-    metadata: VectorRecord["metadata"];
-}
 interface PageRecord {
     url: string;
     title: string;
@@ -311,48 +281,26 @@ interface PageRecord {
     depth: number;
     tags: string[];
     indexedAt: string;
+    summary?: string;
+    description?: string;
+    keywords?: string[];
+}
+interface PageHit {
+    id: string;
+    score: number;
+    title: string;
+    url: string;
+    description: string;
+    tags: string[];
+    depth: number;
+    incomingLinks: number;
+    routeFile: string;
 }
 interface ScopeInfo {
     projectId: string;
     scopeName: string;
-    modelId: string;
     lastIndexedAt: string;
-    vectorCount?: number;
-    lastEstimateTokens?: number;
-    lastEstimateCostUSD?: number;
-    lastEstimateChangedChunks?: number;
-}
-interface VectorStore {
-    upsert(records: VectorRecord[], scope: Scope): Promise<void>;
-    query(queryVector: number[], opts: QueryOpts, scope: Scope): Promise<VectorHit[]>;
-    deleteByIds(ids: string[], scope: Scope): Promise<void>;
-    deleteScope(scope: Scope): Promise<void>;
-    listScopes(scopeProjectId: string): Promise<ScopeInfo[]>;
-    recordScope(info: ScopeInfo): Promise<void>;
-    health(): Promise<{
-        ok: boolean;
-        details?: string;
-    }>;
-    getContentHashes(scope: Scope): Promise<Map<string, string>>;
-    upsertPages(pages: PageRecord[], scope: Scope): Promise<void>;
-    getPage(url: string, scope: Scope): Promise<PageRecord | null>;
-    deletePages(scope: Scope): Promise<void>;
-    getScopeModelId(scope: Scope): Promise<string | null>;
-    dropAllTables(): Promise<void>;
-}
-interface EmbeddingsProvider {
-    embedTexts(texts: string[], modelId: string, task?: string): Promise<number[][]>;
-    estimateTokens(text: string): number;
-}
-interface RerankCandidate {
-    id: string;
-    text: string;
-}
-interface Reranker {
-    rerank(query: string, candidates: RerankCandidate[], topN?: number): Promise<Array<{
-        id: string;
-        score: number;
-    }>>;
+    documentCount?: number;
 }
 interface SearchRequest {
     q: string;
@@ -360,7 +308,6 @@ interface SearchRequest {
     scope?: string;
     pathPrefix?: string;
     tags?: string[];
-    rerank?: boolean;
     groupBy?: "page" | "chunk";
 }
 interface SearchResultChunk {
@@ -384,23 +331,17 @@ interface SearchResponse {
     results: SearchResult[];
     meta: {
         timingsMs: {
-            embed: number;
-            vector: number;
-            rerank: number;
+            search: number;
             total: number;
         };
-        usedRerank: boolean;
-        modelId: string;
     };
 }
 interface IndexStats {
     pagesProcessed: number;
     chunksTotal: number;
     chunksChanged: number;
-    newEmbeddings: number;
+    documentsUpserted: number;
     deletes: number;
-    estimatedTokens: number;
-    estimatedCostUSD: number;
     routeExact: number;
     routeBestEffort: number;
     stageTimingsMs: Record<string, number>;
@@ -416,4 +357,4 @@ interface IndexOptions {
     verbose?: boolean;
 }
 
-export type { Chunk as C, EmbeddingsProvider as E, IndexOptions as I, QueryOpts as Q, ResolvedSearchSocketConfig as R, SearchRequest as S, VectorStore as V, SearchResponse as a, SearchSocketConfig as b, Scope as c, Reranker as d, RerankCandidate as e, IndexStats as f, VectorHit as g, VectorRecord as h };
+export type { Chunk as C, IndexOptions as I, PageHit as P, ResolvedSearchSocketConfig as R, SearchRequest as S, VectorHit as V, SearchResponse as a, SearchResult as b, SearchSocketConfig as c, Scope as d, ScopeInfo as e, PageRecord as f, IndexStats as g };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "searchsocket",
-  "version": "0.3.3",
+  "version": "0.5.0",
   "description": "Semantic site search and MCP retrieval for SvelteKit static sites",
   "license": "MIT",
   "author": "Greg Priday <greg@siteorigin.com>",
@@ -46,6 +46,11 @@
       "types": "./dist/client.d.ts",
       "import": "./dist/client.js",
       "require": "./dist/client.cjs"
+    },
+    "./scroll": {
+      "types": "./dist/scroll.d.ts",
+      "import": "./dist/scroll.js",
+      "require": "./dist/scroll.cjs"
     }
   },
   "scripts": {
@@ -53,15 +58,16 @@
     "clean": "rm -rf dist",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
-    "test:watch": "vitest"
+    "test:watch": "vitest",
+    "test:quality": "SEARCHSOCKET_QUALITY_TESTS=1 vitest run tests/quality.test.ts"
   },
   "engines": {
     "node": ">=20"
   },
   "packageManager": "pnpm@10.29.2",
   "dependencies": {
-    "@libsql/client": "^0.17.0",
     "@modelcontextprotocol/sdk": "^1.26.0",
+    "@upstash/search": "^0.1.7",
     "cheerio": "^1.2.0",
     "chokidar": "^5.0.0",
     "commander": "^14.0.3",
@@ -79,6 +85,7 @@
     "@types/express": "^5.0.6",
     "@types/node": "^25.2.2",
     "@types/turndown": "^5.0.6",
+    "jsdom": "^28.1.0",
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
     "vitest": "^4.0.18"