searchsocket 0.5.0 → 0.6.0

package/dist/{types-Dk43uz25.d.ts → types-029hl6P2.d.ts}

@@ -1,5 +1,10 @@
+type Awaitable<T> = T | Promise<T>;
 type ScopeMode = "fixed" | "git" | "env";
 type SourceMode = "static-output" | "crawl" | "content-files" | "build";
+interface OutgoingLink {
+    url: string;
+    anchorText: string;
+}
 interface SearchSocketConfig {
     project?: {
         id?: string;
@@ -43,6 +48,7 @@ interface SearchSocketConfig {
         dropSelectors?: string[];
         ignoreAttr?: string;
         noindexAttr?: string;
+        imageDescAttr?: string;
         respectRobotsNoindex?: boolean;
     };
     transform?: {
@@ -59,34 +65,47 @@ interface SearchSocketConfig {
         dontSplitInside?: Array<"code" | "table" | "blockquote">;
         prependTitle?: boolean;
         pageSummaryChunk?: boolean;
+        weightHeadings?: boolean;
     };
     upstash?: {
         url?: string;
         token?: string;
         urlEnv?: string;
         tokenEnv?: string;
+        namespaces?: {
+            pages?: string;
+            chunks?: string;
+        };
+    };
+    embedding?: {
+        model?: string;
+        dimensions?: number;
+        taskType?: string;
+        batchSize?: number;
     };
     search?: {
-        semanticWeight?: number;
-        inputEnrichment?: boolean;
-        reranking?: boolean;
         dualSearch?: boolean;
         pageSearchWeight?: number;
     };
     ranking?: {
         enableIncomingLinkBoost?: boolean;
         enableDepthBoost?: boolean;
+        enableFreshnessBoost?: boolean;
+        freshnessDecayRate?: number;
+        enableAnchorTextBoost?: boolean;
         pageWeights?: Record<string, number>;
         aggregationCap?: number;
         aggregationDecay?: number;
         minChunkScoreRatio?: number;
-        minScore?: number;
+        minScoreRatio?: number;
         scoreGapThreshold?: number;
         weights?: {
             incomingLinks?: number;
             depth?: number;
             aggregation?: number;
             titleMatch?: number;
+            freshness?: number;
+            anchorText?: number;
         };
     };
     api?: {
@@ -101,12 +120,28 @@ interface SearchSocketConfig {
     };
     mcp?: {
         enable?: boolean;
+        access?: "public" | "private";
         transport?: "stdio" | "http";
         http?: {
             port?: number;
             path?: string;
+            apiKey?: string;
+            apiKeyEnv?: string;
+        };
+        handle?: {
+            path?: string;
+            apiKey?: string;
+            enableJsonResponse?: boolean;
         };
     };
+    llmsTxt?: {
+        enable?: boolean;
+        outputPath?: string;
+        title?: string;
+        description?: string;
+        generateFull?: boolean;
+        serveMarkdownVariants?: boolean;
+    };
     state?: {
         dir?: string;
     };
@@ -154,6 +189,7 @@ interface ResolvedSearchSocketConfig {
         dropSelectors: string[];
         ignoreAttr: string;
         noindexAttr: string;
+        imageDescAttr: string;
         respectRobotsNoindex: boolean;
     };
     transform: {
@@ -170,34 +206,47 @@ interface ResolvedSearchSocketConfig {
         dontSplitInside: Array<"code" | "table" | "blockquote">;
         prependTitle: boolean;
         pageSummaryChunk: boolean;
+        weightHeadings: boolean;
     };
     upstash: {
         url?: string;
         token?: string;
         urlEnv: string;
         tokenEnv: string;
+        namespaces: {
+            pages: string;
+            chunks: string;
+        };
+    };
+    embedding: {
+        model: string;
+        dimensions: number;
+        taskType: string;
+        batchSize: number;
     };
     search: {
-        semanticWeight: number;
-        inputEnrichment: boolean;
-        reranking: boolean;
         dualSearch: boolean;
         pageSearchWeight: number;
     };
     ranking: {
         enableIncomingLinkBoost: boolean;
         enableDepthBoost: boolean;
+        enableFreshnessBoost: boolean;
+        freshnessDecayRate: number;
+        enableAnchorTextBoost: boolean;
         pageWeights: Record<string, number>;
         aggregationCap: number;
         aggregationDecay: number;
         minChunkScoreRatio: number;
-        minScore: number;
+        minScoreRatio: number;
         scoreGapThreshold: number;
         weights: {
             incomingLinks: number;
             depth: number;
             aggregation: number;
             titleMatch: number;
+            freshness: number;
+            anchorText: number;
         };
     };
     api: {
@@ -212,11 +261,27 @@ interface ResolvedSearchSocketConfig {
     };
     mcp: {
         enable: boolean;
+        access: "public" | "private";
         transport: "stdio" | "http";
         http: {
             port: number;
             path: string;
+            apiKey?: string;
+            apiKeyEnv?: string;
         };
+        handle: {
+            path: string;
+            apiKey?: string;
+            enableJsonResponse: boolean;
+        };
+    };
+    llmsTxt: {
+        enable: boolean;
+        outputPath: string;
+        title?: string;
+        description?: string;
+        generateFull: boolean;
+        serveMarkdownVariants: boolean;
     };
     state: {
         dir: string;
@@ -227,6 +292,19 @@ interface Scope {
     scopeName: string;
     scopeId: string;
 }
+interface ExtractedPage {
+    url: string;
+    title: string;
+    markdown: string;
+    outgoingLinks: OutgoingLink[];
+    noindex: boolean;
+    tags: string[];
+    description?: string;
+    keywords?: string[];
+    weight?: number;
+    publishedAt?: number;
+    meta?: Record<string, string | number | boolean | string[]>;
+}
 interface Chunk {
     chunkKey: string;
     ordinal: number;
@@ -234,6 +312,7 @@ interface Chunk {
     path: string;
     title: string;
     sectionTitle?: string;
+    headingLevel?: number;
     headingPath: string[];
     chunkText: string;
     snippet: string;
@@ -244,6 +323,9 @@ interface Chunk {
     contentHash: string;
     description?: string;
     keywords?: string[];
+    publishedAt?: number;
+    incomingAnchorText?: string;
+    meta?: Record<string, string | number | boolean | string[]>;
 }
 interface VectorHit {
     id: string;
@@ -264,8 +346,13 @@ interface VectorHit {
         incomingLinks: number;
         routeFile: string;
         tags: string[];
+        type?: "chunk" | "page" | "image";
         description?: string;
         keywords?: string[];
+        incomingAnchorText?: string;
+        imageSrc?: string;
+        imageAlt?: string;
+        publishedAt?: number;
     };
 }
 interface PageRecord {
@@ -278,12 +365,16 @@ interface PageRecord {
     routeResolution: "exact" | "best-effort";
     incomingLinks: number;
     outgoingLinks: number;
+    outgoingLinkUrls?: string[];
     depth: number;
     tags: string[];
     indexedAt: string;
     summary?: string;
     description?: string;
     keywords?: string[];
+    contentHash?: string;
+    publishedAt?: number;
+    meta?: Record<string, string | number | boolean | string[]>;
 }
 interface PageHit {
     id: string;
@@ -295,6 +386,7 @@ interface PageHit {
     depth: number;
     incomingLinks: number;
     routeFile: string;
+    publishedAt?: number;
 }
 interface ScopeInfo {
     projectId: string;
@@ -302,17 +394,50 @@ interface ScopeInfo {
     lastIndexedAt: string;
     documentCount?: number;
 }
+interface RankingOverrides {
+    ranking?: {
+        enableIncomingLinkBoost?: boolean;
+        enableDepthBoost?: boolean;
+        aggregationCap?: number;
+        aggregationDecay?: number;
+        minChunkScoreRatio?: number;
+        minScoreRatio?: number;
+        scoreGapThreshold?: number;
+        weights?: {
+            incomingLinks?: number;
+            depth?: number;
+            aggregation?: number;
+            titleMatch?: number;
+        };
+    };
+    search?: {
+        pageSearchWeight?: number;
+    };
+}
 interface SearchRequest {
     q: string;
     topK?: number;
     scope?: string;
     pathPrefix?: string;
     tags?: string[];
+    filters?: Record<string, string | number | boolean>;
     groupBy?: "page" | "chunk";
+    maxSubResults?: number;
+    debug?: boolean;
+    rankingOverrides?: RankingOverrides;
+}
+interface ScoreBreakdown {
+    baseScore: number;
+    incomingLinkBoost: number;
+    depthBoost: number;
+    titleMatchBoost: number;
+    freshnessBoost: number;
+    anchorTextMatchBoost: number;
 }
 interface SearchResultChunk {
     sectionTitle?: string;
     snippet: string;
+    chunkText?: string;
     headingPath: string[];
     score: number;
 }
@@ -321,9 +446,11 @@ interface SearchResult {
     title: string;
     sectionTitle?: string;
     snippet: string;
+    chunkText?: string;
     score: number;
     routeFile: string;
     chunks?: SearchResultChunk[];
+    breakdown?: ScoreBreakdown;
 }
 interface SearchResponse {
     q: string;
@@ -338,6 +465,8 @@ interface SearchResponse {
 }
 interface IndexStats {
     pagesProcessed: number;
+    pagesChanged: number;
+    pagesDeleted: number;
     chunksTotal: number;
     chunksChanged: number;
     documentsUpserted: number;
@@ -346,6 +475,20 @@ interface IndexStats {
     routeBestEffort: number;
     stageTimingsMs: Record<string, number>;
 }
+interface IndexingHooks {
+    transformPage?: (page: ExtractedPage) => Awaitable<ExtractedPage | null>;
+    transformChunk?: (chunk: Chunk) => Awaitable<Chunk | null>;
+    beforeIndex?: (chunks: Chunk[]) => Awaitable<Chunk[]>;
+    afterIndex?: (stats: IndexStats) => Awaitable<void>;
+}
+interface CustomRecord {
+    url: string;
+    title: string;
+    content: string;
+    metadata?: Record<string, string>;
+    tags?: string[];
+    weight?: number;
+}
 interface IndexOptions {
     scopeOverride?: string;
     changedOnly?: boolean;
@@ -355,6 +498,34 @@ interface IndexOptions {
     maxPages?: number;
     maxChunks?: number;
     verbose?: boolean;
+    customRecords?: CustomRecord[];
+}
+interface SiteTreeNode {
+    url: string;
+    title: string;
+    depth: number;
+    routeFile: string;
+    isIndexed: boolean;
+    childCount: number;
+    children: SiteTreeNode[];
+}
+interface SiteStructureResult {
+    root: SiteTreeNode;
+    totalPages: number;
+    truncated: boolean;
+}
+type RelationshipType = "outgoing_link" | "incoming_link" | "sibling" | "semantic";
+interface RelatedPage {
+    url: string;
+    title: string;
+    score: number;
+    relationshipType: RelationshipType;
+    routeFile: string;
+}
+interface RelatedPagesResult {
+    sourceUrl: string;
+    scope: string;
+    relatedPages: RelatedPage[];
 }
 
-export type { Chunk as C, IndexOptions as I, PageHit as P, ResolvedSearchSocketConfig as R, SearchRequest as S, VectorHit as V, SearchResponse as a, SearchResult as b, SearchSocketConfig as c, Scope as d, ScopeInfo as e, PageRecord as f, IndexStats as g };
+export type { Awaitable as A, Chunk as C, IndexingHooks as I, PageHit as P, ResolvedSearchSocketConfig as R, SearchRequest as S, VectorHit as V, SearchResponse as a, SearchResult as b, SearchSocketConfig as c, Scope as d, ScopeInfo as e, PageRecord as f, IndexOptions as g, IndexStats as h, SiteStructureResult as i, RelatedPagesResult as j, CustomRecord as k };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "searchsocket",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Semantic site search and MCP retrieval for SvelteKit static sites",
   "license": "MIT",
   "author": "Greg Priday <greg@siteorigin.com>",
@@ -26,6 +26,7 @@
   "files": [
     "dist",
     "!dist/**/*.map",
+    "src/svelte",
     "README.md"
   ],
   "bin": {
@@ -51,6 +52,19 @@
       "types": "./dist/scroll.d.ts",
       "import": "./dist/scroll.js",
       "require": "./dist/scroll.cjs"
+    },
+    "./svelte": {
+      "types": "./src/svelte/index.svelte.ts",
+      "svelte": "./src/svelte/index.svelte.ts",
+      "default": "./src/svelte/index.svelte.ts"
+    }
+  },
+  "peerDependencies": {
+    "svelte": "^5.0.0"
+  },
+  "peerDependenciesMeta": {
+    "svelte": {
+      "optional": true
     }
   },
   "scripts": {
@@ -66,8 +80,9 @@
   },
   "packageManager": "pnpm@10.29.2",
   "dependencies": {
+    "@clack/prompts": "^1.2.0",
     "@modelcontextprotocol/sdk": "^1.26.0",
-    "@upstash/search": "^0.1.7",
+    "@upstash/vector": "^1.2.3",
     "cheerio": "^1.2.0",
     "chokidar": "^5.0.0",
     "commander": "^14.0.3",
@@ -76,16 +91,19 @@
     "fast-glob": "^3.3.3",
     "gray-matter": "^4.0.3",
     "jiti": "^2.6.1",
+    "magicast": "^0.5.2",
     "p-limit": "^7.3.0",
     "turndown": "^7.2.2",
     "turndown-plugin-gfm": "^1.0.2",
     "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@sveltejs/vite-plugin-svelte": "^6.2.4",
     "@types/express": "^5.0.6",
     "@types/node": "^25.2.2",
     "@types/turndown": "^5.0.6",
     "jsdom": "^28.1.0",
+    "svelte": "^5.55.1",
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
     "vitest": "^4.0.18"

package/src/svelte/SearchSocket.svelte

@@ -0,0 +1,35 @@
+<script lang="ts">
+  import { serializeMetaValue, validateMetaKey } from "../utils/structured-meta";
+  import type { MetaValue } from "../utils/structured-meta";
+
+  interface Props {
+    weight?: number;
+    noindex?: boolean;
+    tags?: string[];
+    meta?: Record<string, MetaValue>;
+  }
+
+  let { weight, noindex, tags, meta }: Props = $props();
+
+  const metaEntries = $derived(
+    meta
+      ? Object.entries(meta).filter(([key]) => validateMetaKey(key))
+      : []
+  );
+</script>
+
+<svelte:head>
+  {#if weight !== undefined}
+    <meta name="searchsocket-weight" content={String(weight)} />
+  {/if}
+  {#if noindex}
+    <meta name="searchsocket:noindex" content="true" />
+  {/if}
+  {#if tags && tags.length > 0}
+    <meta name="searchsocket:tags" content={tags.join(",")} data-type="string[]" />
+  {/if}
+  {#each metaEntries as [key, value]}
+    {@const serialized = serializeMetaValue(value)}
+    <meta name={`searchsocket:${key}`} content={serialized.content} data-type={serialized.dataType} />
+  {/each}
+</svelte:head>

package/src/svelte/index.svelte.ts

@@ -0,0 +1,181 @@
+import type { SearchRequest, SearchResponse, SearchResult } from "../types";
+export { default as SearchSocket } from "./SearchSocket.svelte";
+
+export interface CreateSearchOptions {
+  endpoint?: string;
+  debounce?: number;
+  cache?: boolean;
+  cacheSize?: number;
+  fetchImpl?: typeof fetch;
+  topK?: number;
+  scope?: string;
+  pathPrefix?: string;
+  tags?: string[];
+  filters?: Record<string, string | number | boolean>;
+  groupBy?: "page" | "chunk";
+  maxSubResults?: number;
+}
+
+class LruCache<K, V> {
+  private map = new Map<K, V>();
+  constructor(private maxSize: number) {}
+
+  get(key: K): V | undefined {
+    const value = this.map.get(key);
+    if (value !== undefined) {
+      // Move to end (most recently used)
+      this.map.delete(key);
+      this.map.set(key, value);
+    }
+    return value;
+  }
+
+  set(key: K, value: V): void {
+    this.map.delete(key);
+    this.map.set(key, value);
+    if (this.map.size > this.maxSize) {
+      const oldest = this.map.keys().next().value!;
+      this.map.delete(oldest);
+    }
+  }
+
+  get size(): number {
+    return this.map.size;
+  }
+}
+
+function buildCacheKey(query: string, options: CreateSearchOptions): string {
+  const parts: Record<string, unknown> = { q: query };
+  if (options.topK !== undefined) parts.topK = options.topK;
+  if (options.scope !== undefined) parts.scope = options.scope;
+  if (options.pathPrefix !== undefined) parts.pathPrefix = options.pathPrefix;
+  if (options.tags !== undefined) parts.tags = options.tags;
+  if (options.filters !== undefined) parts.filters = options.filters;
+  if (options.groupBy !== undefined) parts.groupBy = options.groupBy;
+  if (options.maxSubResults !== undefined) parts.maxSubResults = options.maxSubResults;
+  return JSON.stringify(parts);
+}
+
+export interface SearchState {
+  query: string;
+  readonly results: SearchResult[];
+  readonly loading: boolean;
+  readonly error: Error | null;
+  readonly destroy: () => void;
+}
+
+export function createSearch(options: CreateSearchOptions = {}): SearchState {
+  const endpoint = options.endpoint ?? "/api/search";
+  const debounceMs = options.debounce ?? 250;
+  const cacheEnabled = options.cache !== false;
+  const cacheSize = options.cacheSize ?? 50;
+  const fetchFn = options.fetchImpl ?? fetch;
+
+  const resultCache = new LruCache<string, SearchResult[]>(cacheSize);
+
+  let query = $state("");
+  let results = $state<SearchResult[]>([]);
+  let loading = $state(false);
+  let error = $state<Error | null>(null);
+
+  const destroy = $effect.root(() => {
+    $effect(() => {
+      const currentQuery = query;
+
+      if (!currentQuery.trim()) {
+        results = [];
+        loading = false;
+        error = null;
+        return;
+      }
+
+      const cacheKey = buildCacheKey(currentQuery, options);
+
+      if (cacheEnabled) {
+        const cached = resultCache.get(cacheKey);
+        if (cached) {
+          results = cached;
+          loading = false;
+          error = null;
+          return;
+        }
+      }
+
+      loading = true;
+      const controller = new AbortController();
+
+      const timer = setTimeout(async () => {
+        const request: SearchRequest = { q: currentQuery };
+        if (options.topK !== undefined) request.topK = options.topK;
+        if (options.scope !== undefined) request.scope = options.scope;
+        if (options.pathPrefix !== undefined) request.pathPrefix = options.pathPrefix;
+        if (options.tags !== undefined) request.tags = options.tags;
+        if (options.filters !== undefined) request.filters = options.filters;
+        if (options.groupBy !== undefined) request.groupBy = options.groupBy;
+        if (options.maxSubResults !== undefined) request.maxSubResults = options.maxSubResults;
+
+        try {
+          const response = await fetchFn(endpoint, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(request),
+            signal: controller.signal,
+          });
+
+          let payload: unknown;
+          try {
+            payload = await response.json();
+          } catch {
+            throw new Error(response.ok ? "Invalid search response" : "Search failed");
+          }
+
+          if (!response.ok) {
+            const message =
+              (payload as { error?: { message?: string } }).error?.message ?? "Search failed";
+            throw new Error(message);
+          }
+
+          const data = payload as SearchResponse;
+          if (cacheEnabled) {
+            resultCache.set(cacheKey, data.results);
+          }
+          results = data.results;
+          error = null;
+        } catch (err) {
+          if (err instanceof DOMException && err.name === "AbortError") return;
+          if (controller.signal.aborted) return;
+          error = err instanceof Error ? err : new Error(String(err));
+          results = [];
+        } finally {
+          if (!controller.signal.aborted) {
+            loading = false;
+          }
+        }
+      }, debounceMs);
+
+      return () => {
+        clearTimeout(timer);
+        controller.abort();
+      };
+    });
+  });
+
+  return {
+    get query() {
+      return query;
+    },
+    set query(v: string) {
+      query = v;
+    },
+    get results() {
+      return results;
+    },
+    get loading() {
+      return loading;
+    },
+    get error() {
+      return error;
+    },
+    destroy,
+  };
+}