searchsocket 0.4.0 → 0.6.0

package/dist/{types-z2dw3H6E.d.ts → types-029hl6P2.d.ts}

@@ -1,6 +1,10 @@
+type Awaitable<T> = T | Promise<T>;
 type ScopeMode = "fixed" | "git" | "env";
 type SourceMode = "static-output" | "crawl" | "content-files" | "build";
-type EmbeddingProvider = "jina";
+interface OutgoingLink {
+    url: string;
+    anchorText: string;
+}
 interface SearchSocketConfig {
     project?: {
         id?: string;
@@ -44,6 +48,7 @@ interface SearchSocketConfig {
         dropSelectors?: string[];
         ignoreAttr?: string;
         noindexAttr?: string;
+        imageDescAttr?: string;
         respectRobotsNoindex?: boolean;
     };
     transform?: {
@@ -60,44 +65,47 @@ interface SearchSocketConfig {
         dontSplitInside?: Array<"code" | "table" | "blockquote">;
         prependTitle?: boolean;
         pageSummaryChunk?: boolean;
+        weightHeadings?: boolean;
     };
-    embeddings?: {
-        provider?: EmbeddingProvider;
-        model?: string;
-        apiKey?: string;
-        apiKeyEnv?: string;
-        batchSize?: number;
-        concurrency?: number;
-        pricePer1kTokens?: number;
-    };
-    vector?: {
-        dimension?: number;
-        turso?: {
-            url?: string;
-            authToken?: string;
-            urlEnv?: string;
-            authTokenEnv?: string;
-            localPath?: string;
+    upstash?: {
+        url?: string;
+        token?: string;
+        urlEnv?: string;
+        tokenEnv?: string;
+        namespaces?: {
+            pages?: string;
+            chunks?: string;
         };
     };
-    rerank?: {
-        enabled?: boolean;
-        topN?: number;
+    embedding?: {
         model?: string;
+        dimensions?: number;
+        taskType?: string;
+        batchSize?: number;
+    };
+    search?: {
+        dualSearch?: boolean;
+        pageSearchWeight?: number;
     };
     ranking?: {
         enableIncomingLinkBoost?: boolean;
         enableDepthBoost?: boolean;
+        enableFreshnessBoost?: boolean;
+        freshnessDecayRate?: number;
+        enableAnchorTextBoost?: boolean;
         pageWeights?: Record<string, number>;
         aggregationCap?: number;
         aggregationDecay?: number;
         minChunkScoreRatio?: number;
-        minScore?: number;
+        minScoreRatio?: number;
+        scoreGapThreshold?: number;
         weights?: {
             incomingLinks?: number;
             depth?: number;
-            rerank?: number;
             aggregation?: number;
+            titleMatch?: number;
+            freshness?: number;
+            anchorText?: number;
         };
     };
     api?: {
@@ -112,15 +120,30 @@ interface SearchSocketConfig {
     };
     mcp?: {
         enable?: boolean;
+        access?: "public" | "private";
         transport?: "stdio" | "http";
         http?: {
             port?: number;
             path?: string;
+            apiKey?: string;
+            apiKeyEnv?: string;
         };
+        handle?: {
+            path?: string;
+            apiKey?: string;
+            enableJsonResponse?: boolean;
+        };
+    };
+    llmsTxt?: {
+        enable?: boolean;
+        outputPath?: string;
+        title?: string;
+        description?: string;
+        generateFull?: boolean;
+        serveMarkdownVariants?: boolean;
     };
     state?: {
         dir?: string;
-        writeMirror?: boolean;
     };
 }
 interface ResolvedSearchSocketConfig {
@@ -166,6 +189,7 @@ interface ResolvedSearchSocketConfig {
         dropSelectors: string[];
         ignoreAttr: string;
         noindexAttr: string;
+        imageDescAttr: string;
         respectRobotsNoindex: boolean;
     };
     transform: {
@@ -182,44 +206,47 @@ interface ResolvedSearchSocketConfig {
         dontSplitInside: Array<"code" | "table" | "blockquote">;
         prependTitle: boolean;
         pageSummaryChunk: boolean;
+        weightHeadings: boolean;
     };
-    embeddings: {
-        provider: EmbeddingProvider;
-        model: string;
-        apiKey?: string;
-        apiKeyEnv: string;
-        batchSize: number;
-        concurrency: number;
-        pricePer1kTokens?: number;
-    };
-    vector: {
-        dimension?: number;
-        turso: {
-            url?: string;
-            authToken?: string;
-            urlEnv: string;
-            authTokenEnv: string;
-            localPath: string;
+    upstash: {
+        url?: string;
+        token?: string;
+        urlEnv: string;
+        tokenEnv: string;
+        namespaces: {
+            pages: string;
+            chunks: string;
         };
     };
-    rerank: {
-        enabled: boolean;
-        topN: number;
+    embedding: {
         model: string;
+        dimensions: number;
+        taskType: string;
+        batchSize: number;
+    };
+    search: {
+        dualSearch: boolean;
+        pageSearchWeight: number;
     };
     ranking: {
         enableIncomingLinkBoost: boolean;
         enableDepthBoost: boolean;
+        enableFreshnessBoost: boolean;
+        freshnessDecayRate: number;
+        enableAnchorTextBoost: boolean;
         pageWeights: Record<string, number>;
         aggregationCap: number;
         aggregationDecay: number;
         minChunkScoreRatio: number;
-        minScore: number;
+        minScoreRatio: number;
+        scoreGapThreshold: number;
         weights: {
             incomingLinks: number;
             depth: number;
-            rerank: number;
             aggregation: number;
+            titleMatch: number;
+            freshness: number;
+            anchorText: number;
         };
     };
     api: {
@@ -234,15 +261,30 @@ interface ResolvedSearchSocketConfig {
     };
     mcp: {
         enable: boolean;
+        access: "public" | "private";
         transport: "stdio" | "http";
         http: {
             port: number;
             path: string;
+            apiKey?: string;
+            apiKeyEnv?: string;
+        };
+        handle: {
+            path: string;
+            apiKey?: string;
+            enableJsonResponse: boolean;
         };
     };
+    llmsTxt: {
+        enable: boolean;
+        outputPath: string;
+        title?: string;
+        description?: string;
+        generateFull: boolean;
+        serveMarkdownVariants: boolean;
+    };
     state: {
         dir: string;
-        writeMirror: boolean;
     };
 }
 interface Scope {
@@ -250,6 +292,19 @@ interface Scope {
     scopeName: string;
     scopeId: string;
 }
+interface ExtractedPage {
+    url: string;
+    title: string;
+    markdown: string;
+    outgoingLinks: OutgoingLink[];
+    noindex: boolean;
+    tags: string[];
+    description?: string;
+    keywords?: string[];
+    weight?: number;
+    publishedAt?: number;
+    meta?: Record<string, string | number | boolean | string[]>;
+}
 interface Chunk {
     chunkKey: string;
     ordinal: number;
@@ -257,6 +312,7 @@ interface Chunk {
     path: string;
     title: string;
     sectionTitle?: string;
+    headingLevel?: number;
     headingPath: string[];
     chunkText: string;
     snippet: string;
@@ -267,10 +323,13 @@ interface Chunk {
     contentHash: string;
     description?: string;
     keywords?: string[];
+    publishedAt?: number;
+    incomingAnchorText?: string;
+    meta?: Record<string, string | number | boolean | string[]>;
 }
-interface VectorRecord {
+interface VectorHit {
     id: string;
-    vector: number[];
+    score: number;
     metadata: {
         projectId: string;
         scopeName: string;
@@ -283,25 +342,19 @@ interface VectorRecord {
         chunkText: string;
         ordinal: number;
         contentHash: string;
-        modelId: string;
         depth: number;
         incomingLinks: number;
         routeFile: string;
         tags: string[];
+        type?: "chunk" | "page" | "image";
         description?: string;
         keywords?: string[];
+        incomingAnchorText?: string;
+        imageSrc?: string;
+        imageAlt?: string;
+        publishedAt?: number;
     };
 }
-interface QueryOpts {
-    topK: number;
-    pathPrefix?: string;
-    tags?: string[];
-}
-interface VectorHit {
-    id: string;
-    score: number;
-    metadata: VectorRecord["metadata"];
-}
 interface PageRecord {
     url: string;
     title: string;
@@ -312,51 +365,54 @@ interface PageRecord {
     routeResolution: "exact" | "best-effort";
     incomingLinks: number;
     outgoingLinks: number;
+    outgoingLinkUrls?: string[];
     depth: number;
     tags: string[];
     indexedAt: string;
+    summary?: string;
+    description?: string;
+    keywords?: string[];
+    contentHash?: string;
+    publishedAt?: number;
+    meta?: Record<string, string | number | boolean | string[]>;
+}
+interface PageHit {
+    id: string;
+    score: number;
+    title: string;
+    url: string;
+    description: string;
+    tags: string[];
+    depth: number;
+    incomingLinks: number;
+    routeFile: string;
+    publishedAt?: number;
 }
 interface ScopeInfo {
     projectId: string;
     scopeName: string;
-    modelId: string;
     lastIndexedAt: string;
-    vectorCount?: number;
-    lastEstimateTokens?: number;
-    lastEstimateCostUSD?: number;
-    lastEstimateChangedChunks?: number;
-}
-interface VectorStore {
-    upsert(records: VectorRecord[], scope: Scope): Promise<void>;
-    query(queryVector: number[], opts: QueryOpts, scope: Scope): Promise<VectorHit[]>;
-    deleteByIds(ids: string[], scope: Scope): Promise<void>;
-    deleteScope(scope: Scope): Promise<void>;
-    listScopes(scopeProjectId: string): Promise<ScopeInfo[]>;
-    recordScope(info: ScopeInfo): Promise<void>;
-    health(): Promise<{
-        ok: boolean;
-        details?: string;
-    }>;
-    getContentHashes(scope: Scope): Promise<Map<string, string>>;
-    upsertPages(pages: PageRecord[], scope: Scope): Promise<void>;
-    getPage(url: string, scope: Scope): Promise<PageRecord | null>;
-    deletePages(scope: Scope): Promise<void>;
-    getScopeModelId(scope: Scope): Promise<string | null>;
-    dropAllTables(): Promise<void>;
+    documentCount?: number;
 }
-interface EmbeddingsProvider {
-    embedTexts(texts: string[], modelId: string, task?: string): Promise<number[][]>;
-    estimateTokens(text: string): number;
-}
-interface RerankCandidate {
-    id: string;
-    text: string;
-}
-interface Reranker {
-    rerank(query: string, candidates: RerankCandidate[], topN?: number): Promise<Array<{
-        id: string;
-        score: number;
-    }>>;
+interface RankingOverrides {
+    ranking?: {
+        enableIncomingLinkBoost?: boolean;
+        enableDepthBoost?: boolean;
+        aggregationCap?: number;
+        aggregationDecay?: number;
+        minChunkScoreRatio?: number;
+        minScoreRatio?: number;
+        scoreGapThreshold?: number;
+        weights?: {
+            incomingLinks?: number;
+            depth?: number;
+            aggregation?: number;
+            titleMatch?: number;
+        };
+    };
+    search?: {
+        pageSearchWeight?: number;
+    };
 }
 interface SearchRequest {
     q: string;
@@ -364,13 +420,24 @@ interface SearchRequest {
     scope?: string;
     pathPrefix?: string;
     tags?: string[];
-    rerank?: boolean;
+    filters?: Record<string, string | number | boolean>;
     groupBy?: "page" | "chunk";
-    stream?: boolean;
+    maxSubResults?: number;
+    debug?: boolean;
+    rankingOverrides?: RankingOverrides;
+}
+interface ScoreBreakdown {
+    baseScore: number;
+    incomingLinkBoost: number;
+    depthBoost: number;
+    titleMatchBoost: number;
+    freshnessBoost: number;
+    anchorTextMatchBoost: number;
 }
 interface SearchResultChunk {
     sectionTitle?: string;
     snippet: string;
+    chunkText?: string;
     headingPath: string[];
     score: number;
 }
@@ -379,9 +446,11 @@ interface SearchResult {
     title: string;
     sectionTitle?: string;
     snippet: string;
+    chunkText?: string;
     score: number;
     routeFile: string;
     chunks?: SearchResultChunk[];
+    breakdown?: ScoreBreakdown;
 }
 interface SearchResponse {
     q: string;
@@ -389,27 +458,37 @@ interface SearchResponse {
     results: SearchResult[];
     meta: {
         timingsMs: {
-            embed: number;
-            vector: number;
-            rerank: number;
+            search: number;
             total: number;
         };
-        usedRerank: boolean;
-        modelId: string;
     };
 }
 interface IndexStats {
     pagesProcessed: number;
+    pagesChanged: number;
+    pagesDeleted: number;
     chunksTotal: number;
     chunksChanged: number;
-    newEmbeddings: number;
+    documentsUpserted: number;
     deletes: number;
-    estimatedTokens: number;
-    estimatedCostUSD: number;
     routeExact: number;
     routeBestEffort: number;
     stageTimingsMs: Record<string, number>;
 }
+interface IndexingHooks {
+    transformPage?: (page: ExtractedPage) => Awaitable<ExtractedPage | null>;
+    transformChunk?: (chunk: Chunk) => Awaitable<Chunk | null>;
+    beforeIndex?: (chunks: Chunk[]) => Awaitable<Chunk[]>;
+    afterIndex?: (stats: IndexStats) => Awaitable<void>;
+}
+interface CustomRecord {
+    url: string;
+    title: string;
+    content: string;
+    metadata?: Record<string, string>;
+    tags?: string[];
+    weight?: number;
+}
 interface IndexOptions {
     scopeOverride?: string;
     changedOnly?: boolean;
@@ -419,37 +498,34 @@ interface IndexOptions {
     maxPages?: number;
     maxChunks?: number;
     verbose?: boolean;
+    customRecords?: CustomRecord[];
 }
-interface StreamSearchEvent {
-    phase: "initial" | "reranked";
-    data: SearchResponse;
+interface SiteTreeNode {
+    url: string;
+    title: string;
+    depth: number;
+    routeFile: string;
+    isIndexed: boolean;
+    childCount: number;
+    children: SiteTreeNode[];
 }
-interface StreamSearchErrorEvent {
-    phase: "error";
-    data: {
-        error: {
-            code: string;
-            message: string;
-        };
-    };
+interface SiteStructureResult {
+    root: SiteTreeNode;
+    totalPages: number;
+    truncated: boolean;
 }
-type StreamEvent = StreamSearchEvent | StreamSearchErrorEvent;
-interface MergeSearchOptions {
-    /**
-     * If any single result moved more than this many positions, adopt
-     * the reranked order. The reranker is semantic — if it strongly
-     * disagrees with vector similarity on even one result, trust it.
-     * @default 3
-     */
-    maxDisplacement?: number;
+type RelationshipType = "outgoing_link" | "incoming_link" | "sibling" | "semantic";
+interface RelatedPage {
+    url: string;
+    title: string;
+    score: number;
+    relationshipType: RelationshipType;
+    routeFile: string;
 }
-interface MergeSearchResult {
-    response: SearchResponse;
-    usedRerankedOrder: boolean;
-    displacements: Array<{
-        url: string;
-        displacement: number;
-    }>;
+interface RelatedPagesResult {
+    sourceUrl: string;
+    scope: string;
+    relatedPages: RelatedPage[];
 }
 
-export type { Chunk as C, EmbeddingsProvider as E, IndexOptions as I, MergeSearchOptions as M, QueryOpts as Q, ResolvedSearchSocketConfig as R, SearchResponse as S, VectorStore as V, MergeSearchResult as a, SearchRequest as b, StreamSearchEvent as c, SearchSocketConfig as d, Scope as e, Reranker as f, RerankCandidate as g, IndexStats as h, StreamEvent as i, StreamSearchErrorEvent as j, VectorHit as k, VectorRecord as l };
+export type { Awaitable as A, Chunk as C, IndexingHooks as I, PageHit as P, ResolvedSearchSocketConfig as R, SearchRequest as S, VectorHit as V, SearchResponse as a, SearchResult as b, SearchSocketConfig as c, Scope as d, ScopeInfo as e, PageRecord as f, IndexOptions as g, IndexStats as h, SiteStructureResult as i, RelatedPagesResult as j, CustomRecord as k };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "searchsocket",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "Semantic site search and MCP retrieval for SvelteKit static sites",
   "license": "MIT",
   "author": "Greg Priday <greg@siteorigin.com>",
@@ -26,6 +26,7 @@
   "files": [
     "dist",
     "!dist/**/*.map",
+    "src/svelte",
     "README.md"
   ],
   "bin": {
@@ -46,6 +47,24 @@
       "types": "./dist/client.d.ts",
       "import": "./dist/client.js",
       "require": "./dist/client.cjs"
+    },
+    "./scroll": {
+      "types": "./dist/scroll.d.ts",
+      "import": "./dist/scroll.js",
+      "require": "./dist/scroll.cjs"
+    },
+    "./svelte": {
+      "types": "./src/svelte/index.svelte.ts",
+      "svelte": "./src/svelte/index.svelte.ts",
+      "default": "./src/svelte/index.svelte.ts"
+    }
+  },
+  "peerDependencies": {
+    "svelte": "^5.0.0"
+  },
+  "peerDependenciesMeta": {
+    "svelte": {
+      "optional": true
     }
   },
   "scripts": {
@@ -53,15 +72,17 @@
     "clean": "rm -rf dist",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
-    "test:watch": "vitest"
+    "test:watch": "vitest",
+    "test:quality": "SEARCHSOCKET_QUALITY_TESTS=1 vitest run tests/quality.test.ts"
   },
   "engines": {
     "node": ">=20"
   },
   "packageManager": "pnpm@10.29.2",
   "dependencies": {
-    "@libsql/client": "^0.17.0",
+    "@clack/prompts": "^1.2.0",
     "@modelcontextprotocol/sdk": "^1.26.0",
+    "@upstash/vector": "^1.2.3",
     "cheerio": "^1.2.0",
     "chokidar": "^5.0.0",
     "commander": "^14.0.3",
@@ -70,15 +91,19 @@
     "fast-glob": "^3.3.3",
     "gray-matter": "^4.0.3",
     "jiti": "^2.6.1",
+    "magicast": "^0.5.2",
     "p-limit": "^7.3.0",
     "turndown": "^7.2.2",
     "turndown-plugin-gfm": "^1.0.2",
     "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@sveltejs/vite-plugin-svelte": "^6.2.4",
     "@types/express": "^5.0.6",
     "@types/node": "^25.2.2",
     "@types/turndown": "^5.0.6",
+    "jsdom": "^28.1.0",
+    "svelte": "^5.55.1",
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
     "vitest": "^4.0.18"

package/src/svelte/SearchSocket.svelte

@@ -0,0 +1,35 @@
+<script lang="ts">
+  import { serializeMetaValue, validateMetaKey } from "../utils/structured-meta";
+  import type { MetaValue } from "../utils/structured-meta";
+
+  interface Props {
+    weight?: number;
+    noindex?: boolean;
+    tags?: string[];
+    meta?: Record<string, MetaValue>;
+  }
+
+  let { weight, noindex, tags, meta }: Props = $props();
+
+  const metaEntries = $derived(
+    meta
+      ? Object.entries(meta).filter(([key]) => validateMetaKey(key))
+      : []
+  );
+</script>
+
+<svelte:head>
+  {#if weight !== undefined}
+    <meta name="searchsocket-weight" content={String(weight)} />
+  {/if}
+  {#if noindex}
+    <meta name="searchsocket:noindex" content="true" />
+  {/if}
+  {#if tags && tags.length > 0}
+    <meta name="searchsocket:tags" content={tags.join(",")} data-type="string[]" />
+  {/if}
+  {#each metaEntries as [key, value]}
+    {@const serialized = serializeMetaValue(value)}
+    <meta name={`searchsocket:${key}`} content={serialized.content} data-type={serialized.dataType} />
+  {/each}
+</svelte:head>