recallx 1.0.0

package/app/server/retrieval.js

@@ -0,0 +1,486 @@
+import { appendCurrentTelemetryDetails } from "./observability.js";
+import { computeUsageBonus, relationTypeSpecificityBonus } from "./relation-scoring.js";
+const neighborhoodRetrievalRankWeights = {
+    canonicalBase: 2,
+    canonicalSpecificityMultiplier: 1,
+    canonicalUsageMultiplier: 1,
+    inferredBaseMultiplier: 1,
+    inferredSpecificityMultiplier: 1,
+    inferredUsageMultiplier: 1
+};
+const boostedRelationRankWeights = {
+    canonicalBase: 70,
+    canonicalSpecificityMultiplier: 100,
+    canonicalUsageMultiplier: 60,
+    inferredBaseMultiplier: 35,
+    inferredSpecificityMultiplier: 35,
+    inferredUsageMultiplier: 35
+};
+const semanticCandidateMinSimilarity = 0.2;
+const semanticCandidateMaxBonus = 18;
+function resolveSemanticAugmentationSettings(settings) {
+    return {
+        minSimilarity: typeof settings?.minSimilarity === "number" && Number.isFinite(settings.minSimilarity)
+            ? Math.min(Math.max(settings.minSimilarity, 0), 1)
+            : semanticCandidateMinSimilarity,
+        maxBonus: typeof settings?.maxBonus === "number" && Number.isFinite(settings.maxBonus)
+            ? Math.max(settings.maxBonus, 0)
+            : semanticCandidateMaxBonus
+    };
+}
+function prioritizeItems(items, preset, maxItems, bonuses) {
+    const weighted = items
+        .map((item) => ({
+        item,
+        score: scoreItem(item, preset) + (bonuses?.get(item.id) ?? 0)
+    }))
+        .sort((left, right) => right.score - left.score || right.item.updatedAt.localeCompare(left.item.updatedAt))
+        .map(({ item }) => item);
+    return weighted.slice(0, maxItems);
+}
+function buildNeighborhoodResult(repository, nodeId, options) {
+    const canonicalItems = repository.listRelatedNodes(nodeId, 1, options?.relationTypes).map(({ node, relation }) => ({
+        node,
+        edge: {
+            relationId: relation.id,
+            relationType: relation.relationType,
+            relationSource: "canonical",
+            relationStatus: relation.status,
+            relationScore: null,
+            retrievalRank: null,
+            generator: null,
+            reason: `Related via ${relation.relationType}`,
+            direction: relation.fromNodeId === nodeId ? "outgoing" : "incoming",
+            hop: 1
+        }
+    }));
+    const seenNodeIds = new Set(canonicalItems.map((item) => item.node.id));
+    const inferredItems = options?.includeInferred && options.maxInferred
+        ? (() => {
+            const relations = repository
+                .listInferredRelationsForNode(nodeId, Math.max(options.maxInferred * 3, options.maxInferred))
+                .filter((relation) => !options.relationTypes?.length || options.relationTypes.includes(relation.relationType));
+            const relatedNodeIds = relations.map((relation) => relation.fromNodeId === nodeId ? relation.toNodeId : relation.fromNodeId);
+            const relatedNodes = repository.getNodesByIds(relatedNodeIds);
+            return relations
+                .flatMap((relation) => {
+                const relatedNodeId = relation.fromNodeId === nodeId ? relation.toNodeId : relation.fromNodeId;
+                const node = relatedNodes.get(relatedNodeId);
+                if (!node) {
+                    return [];
+                }
+                return [{
+                        node,
+                        edge: {
+                            relationId: relation.id,
+                            relationType: relation.relationType,
+                            relationSource: "inferred",
+                            relationStatus: relation.status,
+                            relationScore: relation.finalScore,
+                            retrievalRank: relation.finalScore,
+                            generator: relation.generator,
+                            reason: `Inferred via ${relation.relationType} (score ${relation.finalScore.toFixed(2)})`,
+                            direction: relation.fromNodeId === nodeId ? "outgoing" : "incoming",
+                            hop: 1
+                        }
+                    }];
+            })
+                .filter((item) => {
+                if (seenNodeIds.has(item.node.id)) {
+                    return false;
+                }
+                seenNodeIds.add(item.node.id);
+                return true;
+            });
+        })()
+        : [];
+    const usageSummaries = repository.getRelationUsageSummaries([...canonicalItems, ...inferredItems].map((item) => item.edge.relationId));
+    const rankedCanonical = rankNeighborhoodItems(canonicalItems, usageSummaries, neighborhoodRetrievalRankWeights);
+    const rankedInferred = options?.includeInferred && options.maxInferred
+        ? rankNeighborhoodItems(inferredItems, usageSummaries, neighborhoodRetrievalRankWeights, options.maxInferred)
+        : [];
+    return {
+        items: [...rankedCanonical, ...rankedInferred],
+        usageSummaries
+    };
+}
+function matchesSearchResultFilters(item, filters) {
+    const typeMatches = !filters.types?.length || filters.types.includes(item.type);
+    const statusMatches = !filters.status?.length || filters.status.includes(item.status);
+    return typeMatches && statusMatches;
+}
+function rankNeighborhoodItems(items, usageSummaries, weights, maxItems) {
+    const ranked = items
+        .map((item) => {
+        const summary = usageSummaries.get(item.edge.relationId);
+        const rank = computeRelationRetrievalRank(item.edge, summary, weights);
+        return {
+            item: {
+                ...item,
+                edge: {
+                    ...item.edge,
+                    reason: formatRelationReason(item.edge.reason, summary)
+                }
+            },
+            rank
+        };
+    })
+        .sort((left, right) => right.rank - left.rank);
+    return (typeof maxItems === "number" ? ranked.slice(0, maxItems) : ranked).map((entry) => ({
+        ...entry.item,
+        edge: {
+            ...entry.item.edge,
+            retrievalRank: entry.rank
+        }
+    }));
+}
+function scoreItem(item, preset) {
+    let score = 0;
+    if (item.canonicality === "canonical")
+        score += 30;
+    if (item.status === "active")
+        score += 10;
+    if (preset === "for-coding") {
+        if (item.type === "project")
+            score += 40;
+        if (item.type === "decision")
+            score += 25;
+        if (item.type === "reference")
+            score += 20;
+    }
+    if (preset === "for-research") {
+        if (item.type === "reference")
+            score += 35;
+        if (item.type === "idea")
+            score += 20;
+        if (item.type === "question")
+            score += 20;
+    }
+    if (preset === "for-assistant") {
+        if (item.type === "project")
+            score += 25;
+        if (item.type === "note")
+            score += 20;
+        if (item.type === "question")
+            score += 10;
+    }
+    return score;
+}
+export function computeRelationRetrievalRank(edge, summary, weights = neighborhoodRetrievalRankWeights) {
+    const usageBonus = computeUsageBonus(summary);
+    const specificityBonus = relationTypeSpecificityBonus(edge.relationType);
+    if (edge.relationSource === "canonical") {
+        return weights.canonicalBase + specificityBonus * weights.canonicalSpecificityMultiplier + usageBonus * weights.canonicalUsageMultiplier;
+    }
+    return ((edge.relationScore ?? 0) * weights.inferredBaseMultiplier +
+        specificityBonus * weights.inferredSpecificityMultiplier +
+        usageBonus * weights.inferredUsageMultiplier);
+}
+export function computeRankCandidateScore(node, query, preset, relationRetrievalRank = 0) {
+    const normalizedQuery = query.toLowerCase();
+    return ((node.title?.toLowerCase().includes(normalizedQuery) ? 50 : 0) +
+        (node.summary?.toLowerCase().includes(normalizedQuery) ? 20 : 0) +
+        (preset === "for-coding" && node.type === "decision" ? 15 : 0) +
+        (node.canonicality === "canonical" ? 10 : 0) +
+        relationRetrievalRank);
+}
+export function shouldUseSemanticCandidateAugmentation(query, candidates) {
+    const normalizedQuery = query.trim().toLowerCase();
+    if (normalizedQuery.length < 6) {
+        return false;
+    }
+    return !candidates.some((candidate) => {
+        const title = candidate.title?.toLowerCase() ?? "";
+        const summary = candidate.summary?.toLowerCase() ?? "";
+        return title.includes(normalizedQuery) || summary.includes(normalizedQuery);
+    });
+}
+export function buildSemanticCandidateBonusMap(semanticMatches, settings) {
+    const resolved = resolveSemanticAugmentationSettings(settings);
+    return new Map([...semanticMatches.entries()]
+        .filter(([, match]) => Number.isFinite(match.similarity) && match.similarity >= resolved.minSimilarity)
+        .map(([nodeId, match]) => {
+        const normalizedSimilarity = resolved.minSimilarity >= 1
+            ? 0
+            : Math.min(1, Math.max(0, match.similarity - resolved.minSimilarity) / (1 - resolved.minSimilarity));
+        const retrievalRank = Number((normalizedSimilarity * resolved.maxBonus).toFixed(4));
+        return [
+            nodeId,
+            {
+                retrievalRank,
+                semanticSimilarity: Number(match.similarity.toFixed(4)),
+                reason: `Semantic similarity ${match.similarity.toFixed(2)} via local-ngram across ${match.matchedChunks} chunk${match.matchedChunks === 1 ? "" : "s"}`
+            }
+        ];
+    }));
+}
+function computeBundleRelationBoost(item, summary) {
+    return computeRelationRetrievalRank(item.edge, summary, {
+        canonicalBase: 120,
+        canonicalSpecificityMultiplier: 100,
+        canonicalUsageMultiplier: 80,
+        inferredBaseMultiplier: 40,
+        inferredSpecificityMultiplier: 40,
+        inferredUsageMultiplier: 40
+    });
+}
+function formatRelationReason(baseReason, summary) {
+    const usageBonus = computeUsageBonus(summary);
+    if (!usageBonus) {
+        return baseReason;
+    }
+    const direction = usageBonus > 0 ? "+" : "";
+    return `${baseReason}, usage ${direction}${usageBonus.toFixed(2)}`;
+}
+function searchResultFromNode(node) {
+    return {
+        id: node.id,
+        type: node.type,
+        title: node.title,
+        summary: node.summary,
+        status: node.status,
+        canonicality: node.canonicality,
+        sourceLabel: node.sourceLabel,
+        updatedAt: node.updatedAt,
+        tags: node.tags
+    };
+}
+function buildRetrievalCandidates(target, neighborhood) {
+    const candidates = new Map([[target.id, searchResultFromNode(target)]]);
+    for (const item of neighborhood) {
+        candidates.set(item.node.id, searchResultFromNode(item.node));
+    }
+    return Array.from(candidates.values());
+}
+export function buildNeighborhoodItems(repository, nodeId, options) {
+    return buildNeighborhoodResult(repository, nodeId, options).items;
+}
+export function buildCandidateRelationBonusMap(repository, targetNodeId, candidateNodeIds) {
+    const candidateNodeIdSet = new Set(candidateNodeIds);
+    const { items: neighborhood, usageSummaries } = buildNeighborhoodResult(repository, targetNodeId, {
+        includeInferred: true,
+        maxInferred: Math.max(4, Math.min(candidateNodeIds.length, 10))
+    });
+    return new Map(neighborhood
+        .filter((item) => candidateNodeIdSet.has(item.node.id))
+        .map((item) => [
+        item.node.id,
+        {
+            retrievalRank: computeRelationRetrievalRank(item.edge, usageSummaries.get(item.edge.relationId), boostedRelationRankWeights),
+            relationSource: item.edge.relationSource,
+            relationType: item.edge.relationType,
+            relationScore: item.edge.relationScore,
+            reason: item.edge.reason
+        }
+    ]));
+}
+export function buildTargetRelatedRetrievalItems(repository, targetId, filters, cachedItems) {
+    const candidates = cachedItems ?? (() => {
+        const target = repository.getNode(targetId);
+        return buildRetrievalCandidates(target, buildNeighborhoodItems(repository, target.id, { includeInferred: true, maxInferred: 4 }));
+    })();
+    return candidates.filter((item) => matchesSearchResultFilters(item, filters));
+}
+async function buildWorkspaceContextBundle(repository, input) {
+    const recentNodes = repository
+        .listNodes(Math.max(input.options.maxItems * 3, 18))
+        .filter((item) => item.status !== "archived");
+    const decisions = input.options.includeDecisions
+        ? recentNodes.filter((item) => item.type === "decision" && (item.status === "active" || item.status === "contested"))
+        : [];
+    const openQuestions = input.options.includeOpenQuestions
+        ? recentNodes.filter((item) => item.type === "question" && ["active", "draft", "contested"].includes(item.status))
+        : [];
+    const baseItems = prioritizeItems(recentNodes, input.preset, input.mode === "micro" ? Math.min(input.options.maxItems, 5) : input.options.maxItems);
+    const activityDigest = input.options.includeRecentActivities
+        ? repository
+            .searchActivities({
+            query: "",
+            filters: {},
+            limit: input.mode === "micro" ? 3 : 6,
+            offset: 0,
+            sort: "updated_at"
+        })
+            .items.map((activity) => `${activity.targetNodeTitle ?? activity.targetNodeId} · ${activity.activityType}: ${activity.body ?? "No details"}`)
+        : [];
+    return {
+        target: {
+            type: "workspace",
+            id: "workspace",
+            title: "Workspace context"
+        },
+        mode: input.mode,
+        preset: input.preset,
+        summary: baseItems[0]?.summary ??
+            "Recent workspace context across active nodes, open questions, decisions, and recent activity trails.",
+        items: baseItems.map((item) => ({
+            nodeId: item.id,
+            type: item.type,
+            title: item.title,
+            summary: item.summary,
+            reason: item.type === "project" ? "Recent workspace project context" : `Recent workspace context for ${input.preset}`
+        })),
+        activityDigest,
+        decisions,
+        openQuestions,
+        sources: baseItems.map((item) => ({
+            nodeId: item.id,
+            sourceLabel: item.sourceLabel
+        }))
+    };
+}
+export async function buildContextBundle(repository, input) {
+    if (!input.target?.id) {
+        return buildWorkspaceContextBundle(repository, input);
+    }
+    const target = repository.getNode(input.target.id);
+    const sharedNeighborhood = input.options.includeRelated || input.options.includeDecisions || input.options.includeOpenQuestions
+        ? buildNeighborhoodResult(repository, target.id, {
+            includeInferred: input.options.includeInferred,
+            maxInferred: input.options.maxInferred
+        })
+        : { items: [], usageSummaries: new Map() };
+    const neighborhood = input.options.includeRelated ? sharedNeighborhood.items : [];
+    const related = neighborhood.map((item) => ({
+        nodeId: item.node.id,
+        type: item.node.type,
+        title: item.node.title,
+        summary: item.node.summary,
+        reason: item.edge.reason,
+        relationId: item.edge.relationId,
+        relationType: item.edge.relationType,
+        relationSource: item.edge.relationSource,
+        relationStatus: item.edge.relationStatus,
+        relationScore: item.edge.relationScore ?? undefined,
+        retrievalRank: item.edge.retrievalRank ?? undefined,
+        generator: item.edge.generator
+    }));
+    const retrievalCandidates = buildRetrievalCandidates(target, sharedNeighborhood.items);
+    const decisions = input.options.includeDecisions
+        ? buildTargetRelatedRetrievalItems(repository, target.id, {
+            types: ["decision"],
+            status: ["active", "contested"]
+        }, retrievalCandidates)
+        : [];
+    const openQuestions = input.options.includeOpenQuestions
+        ? buildTargetRelatedRetrievalItems(repository, target.id, {
+            types: ["question"],
+            status: ["active", "draft", "contested"]
+        }, retrievalCandidates)
+        : [];
+    const targetItem = {
+        id: target.id,
+        type: target.type,
+        title: target.title,
+        summary: target.summary,
+        status: target.status,
+        canonicality: target.canonicality,
+        sourceLabel: target.sourceLabel,
+        updatedAt: target.updatedAt,
+        tags: target.tags
+    };
+    const relatedItems = neighborhood.map((item) => ({
+        id: item.node.id,
+        type: item.node.type,
+        title: item.node.title,
+        summary: item.node.summary,
+        status: item.node.status,
+        canonicality: item.node.canonicality,
+        sourceLabel: item.node.sourceLabel,
+        updatedAt: item.node.updatedAt,
+        tags: item.node.tags
+    }));
+    const relationBonuses = new Map(neighborhood.map((item) => [
+        item.node.id,
+        computeBundleRelationBoost(item, sharedNeighborhood.usageSummaries.get(item.edge.relationId))
+    ]));
+    const candidateItems = [targetItem, ...relatedItems, ...decisions, ...openQuestions];
+    const dedupedItems = Array.from(new Map(candidateItems.map((item) => [item.id, item])).values());
+    const semanticQuery = [target.title, target.summary ?? target.body].filter(Boolean).join("\n");
+    const semanticBonuses = shouldUseSemanticCandidateAugmentation(semanticQuery, dedupedItems.filter((item) => item.id !== target.id))
+        ? buildSemanticCandidateBonusMap(await repository.rankSemanticCandidates(semanticQuery, dedupedItems.filter((item) => item.id !== target.id).map((item) => item.id)), repository.getSemanticAugmentationSettings())
+        : new Map();
+    appendCurrentTelemetryDetails({
+        neighborhoodCount: neighborhood.length,
+        relatedCandidateCount: relatedItems.length,
+        decisionCount: decisions.length,
+        openQuestionCount: openQuestions.length,
+        semanticUsed: semanticBonuses.size > 0
+    });
+    const combinedBonuses = new Map();
+    for (const item of dedupedItems) {
+        combinedBonuses.set(item.id, (relationBonuses.get(item.id) ?? 0) + (semanticBonuses.get(item.id)?.retrievalRank ?? 0));
+    }
+    const baseItems = prioritizeItems(dedupedItems, input.preset, input.mode === "micro" ? Math.min(input.options.maxItems, 5) : input.options.maxItems, combinedBonuses);
+    const itemById = new Map(related.map((item) => [item.nodeId, item]));
+    const bundle = {
+        target: {
+            type: target.type,
+            id: target.id,
+            title: target.title
+        },
+        mode: input.mode,
+        preset: input.preset,
+        summary: target.summary ?? "No target summary yet.",
+        items: baseItems.map((item) => ({
+            nodeId: item.id,
+            type: item.type,
+            title: item.title,
+            summary: item.summary,
+            reason: [
+                itemById.get(item.id)?.reason ?? (item.id === target.id ? "Primary target" : `Included for ${input.preset}`),
+                semanticBonuses.get(item.id)?.reason ?? null
+            ]
+                .filter(Boolean)
+                .join("; "),
+            relationId: itemById.get(item.id)?.relationId,
+            relationType: itemById.get(item.id)?.relationType,
+            relationSource: itemById.get(item.id)?.relationSource,
+            relationStatus: itemById.get(item.id)?.relationStatus,
+            relationScore: itemById.get(item.id)?.relationScore,
+            retrievalRank: (itemById.get(item.id)?.retrievalRank ?? 0) + (semanticBonuses.get(item.id)?.retrievalRank ?? 0) || undefined,
+            semanticSimilarity: semanticBonuses.get(item.id)?.semanticSimilarity,
+            generator: itemById.get(item.id)?.generator ?? null
+        })),
+        activityDigest: input.options.includeRecentActivities
+            ? repository
+                .listNodeActivities(target.id, input.mode === "micro" ? 3 : 6)
+                .map((activity) => `${activity.activityType}: ${activity.body ?? "No details"}`)
+            : [],
+        decisions,
+        openQuestions,
+        sources: baseItems.map((item) => ({
+            nodeId: item.id,
+            sourceLabel: item.sourceLabel
+        }))
+    };
+    appendCurrentTelemetryDetails({
+        bundleItemCount: bundle.items.length,
+        bundleSourceCount: bundle.sources.length
+    });
+    return bundle;
+}
+export function bundleAsMarkdown(bundle) {
+    const sections = [
+        `# ${bundle.target.title ?? bundle.target.id}`,
+        "",
+        `Mode: ${bundle.mode}`,
+        `Preset: ${bundle.preset}`,
+        "",
+        "## Summary",
+        bundle.summary,
+        "",
+        "## Items",
+        ...bundle.items.map((item) => `- ${item.title ?? item.nodeId}: ${item.summary ?? "No summary"} (${item.reason})`)
+    ];
+    if (bundle.decisions.length) {
+        sections.push("", "## Decisions", ...bundle.decisions.map((item) => `- ${item.title ?? item.id}: ${item.summary ?? "No summary"}`));
+    }
+    if (bundle.openQuestions.length) {
+        sections.push("", "## Open Questions", ...bundle.openQuestions.map((item) => `- ${item.title ?? item.id}`));
+    }
+    if (bundle.activityDigest.length) {
+        sections.push("", "## Recent Activities", ...bundle.activityDigest.map((item) => `- ${item}`));
+    }
+    return sections.join("\n");
+}

package/app/server/semantic/chunker.js

@@ -0,0 +1,85 @@
+import { checksumText } from "../utils.js";
+function normalizeTagValue(tag) {
+    return tag.trim().toLowerCase().replace(/\s+/g, " ");
+}
+export function normalizeTagList(tags) {
+    return Array.from(new Set(tags.map(normalizeTagValue).filter(Boolean)));
+}
+export function buildSemanticDocumentText(input) {
+    return [
+        input.title?.trim(),
+        input.summary?.trim(),
+        input.tags.length ? `tags: ${normalizeTagList(input.tags).join(", ")}` : null,
+        input.body?.trim(),
+    ]
+        .filter((part) => Boolean(part))
+        .join("\n\n")
+        .trim();
+}
+function estimateTokenCount(text) {
+    if (!text.trim()) {
+        return 0;
+    }
+    return Math.max(1, Math.ceil(text.length / 4));
+}
+function findChunkBoundary(text, startOffset, endOffset) {
+    for (let index = endOffset; index > startOffset + 300; index -= 1) {
+        if (text[index] === " ") {
+            return index + 1;
+        }
+        if (text[index] === "." && text[index + 1] === " ") {
+            return index + 1;
+        }
+        if (text[index] === "\n" && text[index - 1] === "\n") {
+            return index + 1;
+        }
+    }
+    return endOffset;
+}
+export function buildSemanticChunks(text, chunkEnabled) {
+    const normalized = text.trim();
+    if (!normalized) {
+        return [];
+    }
+    if (!chunkEnabled) {
+        return [
+            {
+                ordinal: 0,
+                chunkHash: checksumText(normalized),
+                chunkText: normalized,
+                tokenCount: estimateTokenCount(normalized),
+                startOffset: 0,
+                endOffset: normalized.length,
+            },
+        ];
+    }
+    const maxChars = 1200;
+    const overlapChars = 180;
+    const chunks = [];
+    let startOffset = 0;
+    let ordinal = 0;
+    while (startOffset < normalized.length) {
+        let endOffset = Math.min(startOffset + maxChars, normalized.length);
+        if (endOffset < normalized.length) {
+            endOffset = findChunkBoundary(normalized, startOffset, endOffset);
+        }
+        const chunkText = normalized.slice(startOffset, endOffset).trim();
+        if (!chunkText) {
+            break;
+        }
+        chunks.push({
+            ordinal,
+            chunkHash: checksumText(chunkText),
+            chunkText,
+            tokenCount: estimateTokenCount(chunkText),
+            startOffset,
+            endOffset,
+        });
+        if (endOffset >= normalized.length) {
+            break;
+        }
+        startOffset = Math.max(endOffset - overlapChars, startOffset + 1);
+        ordinal += 1;
+    }
+    return chunks;
+}

package/app/server/semantic/provider.js

@@ -0,0 +1,124 @@
+const LOCAL_NGRAM_DIMENSION = 384;
+const LEGACY_DETERMINISTIC_PROVIDER = "deterministic";
+const LOCAL_NGRAM_PROVIDER = "local-ngram";
+const LOCAL_NGRAM_MODEL = "chargram-v1";
+class LocalNgramEmbeddingProvider {
+    provider;
+    model;
+    constructor(provider = LOCAL_NGRAM_PROVIDER, model = LOCAL_NGRAM_MODEL) {
+        this.provider = provider;
+        this.model = model;
+    }
+    version = "2";
+    async embedBatch(input) {
+        return input.map((item) => ({
+            nodeId: item.nodeId,
+            chunkOrdinal: item.chunkOrdinal,
+            contentHash: item.contentHash,
+            vector: localNgramVector(item.text, LOCAL_NGRAM_DIMENSION),
+            dimension: LOCAL_NGRAM_DIMENSION,
+        }));
+    }
+}
+function normalizeVector(vector) {
+    const magnitude = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
+    if (!Number.isFinite(magnitude) || magnitude === 0) {
+        return vector;
+    }
+    return vector.map((value) => value / magnitude);
+}
+function normalizeText(text) {
+    return text.toLowerCase().replace(/\s+/g, " ").trim();
+}
+function forEachCharacterNgram(text, callback) {
+    const normalized = normalizeText(text);
+    if (!normalized) {
+        return;
+    }
+    const source = ` ${normalized} `;
+    for (let size = 2; size <= 4; size += 1) {
+        for (let index = 0; index <= source.length - size; index += 1) {
+            const gram = source.slice(index, index + size);
+            // Keep boundary grams that contain spaces, but drop grams that are only whitespace.
+            if (gram.trim()) {
+                callback(gram);
+            }
+        }
+    }
+}
+function fnv1a32(value) {
+    let hash = 0x811c9dc5;
+    for (let index = 0; index < value.length; index += 1) {
+        const codePoint = value.charCodeAt(index);
+        hash ^= codePoint & 0xff;
+        hash = Math.imul(hash, 0x01000193);
+        hash ^= codePoint >>> 8;
+        hash = Math.imul(hash, 0x01000193);
+    }
+    return hash >>> 0;
+}
+function avalanche32(value) {
+    let mixed = value >>> 0;
+    mixed ^= mixed >>> 16;
+    mixed = Math.imul(mixed, 0x7feb352d);
+    mixed ^= mixed >>> 15;
+    mixed = Math.imul(mixed, 0x846ca68b);
+    mixed ^= mixed >>> 16;
+    return mixed >>> 0;
+}
+function localNgramVector(text, dimension) {
+    const vector = new Array(dimension).fill(0);
+    let sawGram = false;
+    forEachCharacterNgram(text, (gram) => {
+        sawGram = true;
+        const hash = fnv1a32(gram);
+        const bucket = hash % dimension;
+        const sign = (avalanche32(hash ^ 0x9e3779b9) & 1) === 0 ? 1 : -1;
+        vector[bucket] += sign;
+    });
+    if (!sawGram) {
+        return vector;
+    }
+    for (let index = 0; index < vector.length; index += 1) {
+        const value = vector[index];
+        vector[index] = Math.sign(value) * Math.log1p(Math.abs(value));
+    }
+    return normalizeVector(vector);
+}
+export function resolveSemanticEmbeddingProvider(input) {
+    const normalized = normalizeSemanticProviderConfig(input);
+    if (!normalized.provider || normalized.provider === "disabled" || !normalized.model || normalized.model === "none") {
+        return null;
+    }
+    if (normalized.provider === LOCAL_NGRAM_PROVIDER) {
+        return new LocalNgramEmbeddingProvider();
+    }
+    return null;
+}
+export async function embedSemanticQueryText(input) {
+    const provider = resolveSemanticEmbeddingProvider({
+        provider: input.provider,
+        model: input.model
+    });
+    if (!provider || !input.text.trim()) {
+        return null;
+    }
+    const [result] = await provider.embedBatch([
+        {
+            nodeId: "__query__",
+            chunkOrdinal: 0,
+            contentHash: "__query__",
+            text: input.text,
+        }
+    ]);
+    return result ?? null;
+}
+export function normalizeSemanticProviderConfig(input) {
+    if (input.provider === LEGACY_DETERMINISTIC_PROVIDER) {
+        return {
+            provider: LOCAL_NGRAM_PROVIDER,
+            model: LOCAL_NGRAM_MODEL,
+        };
+    }
+    return input;
+}

package/app/server/semantic/types.js

@@ -0,0 +1 @@
+export {};