pi-mega-compact 0.11.5 → 0.11.6

package/dist/src/queryReformulation/tfidf.js

@@ -0,0 +1,61 @@
+/**
+ * tfidf.ts — TF-IDF term extraction from neighbor documents (S43A-6).
+ *
+ * Extracts the highest-IDF-weighted terms from a set of neighbor documents
+ * for query expansion.
+ *
+ * Term frequency = raw count of the term in the neighbor document (normalized
+ * by document length for cross-doc comparability).
+ * Inverse document frequency = log(N / df) where N = total docs in corpus.
+ *
+ * This is NOT a stopword list: frequent terms like "the" naturally get a low
+ * IDF and rank below rare terms like "jwt" even when both have similar TF.
+ */
+/**
+ * Extract the top-N terms by TF-IDF weight from the given neighbor documents.
+ *
+ * Returns an empty array when:
+ *   - neighbors list is empty
+ *   - totalDocs <= 0
+ *   - no terms survive filtering (all numeric, all single-char, etc.)
+ */
+export function extractExpansionTerms(neighbors, corpus, topTerms) {
+    if (neighbors.length === 0)
+        return [];
+    const { totalDocs, docFreq } = corpus;
+    if (totalDocs <= 0)
+        return [];
+    // Aggregate term frequency across all neighbor documents
+    const termTf = new Map();
+    for (const neighbor of neighbors) {
+        const words = neighbor.text.toLowerCase().split(/\W+/).filter(Boolean);
+        if (words.length === 0)
+            continue;
+        // Normalize TF by document length so longer docs don't dominate
+        const lenNorm = words.length;
+        for (const word of words) {
+            // Skip pure number tokens (short numbers)
+            if (/^\d+$/.test(word) && word.length <= 4)
+                continue;
+            // Skip single characters
+            if (word.length < 2)
+                continue;
+            // Accumulate across neighbors (multiple docs amplify a term's importance)
+            termTf.set(word, (termTf.get(word) ?? 0) + (1 / lenNorm));
+        }
+    }
+    // Compute TF-IDF for each term
+    const scored = [];
+    for (const [term, tf] of termTf) {
+        const df = docFreq(term);
+        // IDF = ln(1 + (N - df + 0.5) / (df + 0.5)) — smooth version
+        const idf = Math.log(1 + (totalDocs - df + 0.5) / (df + 0.5));
+        const tfIdf = tf * idf;
+        if (tfIdf > 0) {
+            scored.push({ term, tfIdf, df });
+        }
+    }
+    // Sort by TF-IDF descending, take top N
+    scored.sort((a, b) => b.tfIdf - a.tfIdf);
+    return scored.slice(0, topTerms);
+}

package/dist/src/queryReformulation/vagueness.js

@@ -0,0 +1,33 @@
+/**
+ * vagueness.ts — heuristic vagueness detection for S43 query reformulation.
+ *
+ * Detects whether a query is "vague" enough to benefit from reformulation.
+ */
+/**
+ * Heuristic: is this query "vague" enough to benefit from reformulation?
+ * Returns true if the query is short, question-like, or lacks specific terms.
+ *
+ * ANY match -> vague:
+ *   a) Total words <= `vagueMinWords` (e.g. "what about auth")
+ *   b) Contains a question word AND total words <= `vagueVeryShortWords`
+ *
+ * NOTE: Misclassification is low-cost: a specific query that's wrongly expanded
+ * just gets a slightly broader search; a vague query that's wrongly skipped
+ * just gets the raw-query search (today's behavior).
+ */
+export function isVagueQuery(query, opts) {
+    const cleaned = query.replace(/[^\w\s]/g, "").trim();
+    if (!cleaned)
+        return true; // empty or punctuation-only query is vague
+    const words = cleaned.split(/\s+/).filter(Boolean);
+    const wordCount = words.length;
+    // Rule a: very short query
+    if (wordCount <= opts.vagueMinWords)
+        return true;
+    // Rule b: contains a question word AND is short-ish
+    const questionWords = /^(what|why|how|when|where|who|which|does|did|is|are|was|were|can|could|would|should)\b/i;
+    const hasQuestionWord = words.some((w) => questionWords.test(w));
+    if (hasQuestionWord && wordCount <= opts.vagueVeryShortWords)
+        return true;
+    return false;
+}

package/dist/src/queryReformulation.js

@@ -0,0 +1,225 @@
+/**
+ * queryReformulation.ts — fully local, zero-LLM query reformulation for vague
+ * recall queries (S43).
+ *
+ * Shell: delegates TF-IDF extraction to queryReformulation/tfidf.ts and
+ * vagueness detection to queryReformulation/vagueness.ts. Orchestrates the
+ * pipeline here.
+ *
+ * Pipeline:
+ *   1. Embed the raw query via the project's default TrigramEmbedder.
+ *   2. Linear-scan the real `context_chunks` store for top-N cosine neighbors.
+ *   3. Extract high-IDF terms from those neighbors (real TF-IDF, no stopword
+ *      list — high-frequency terms naturally get low IDF).
+ *   4. Expand the query with the top-IDF terms.
+ *   5. Re-embed the expanded query and fuse both ranked lists via RRF.
+ *
+ * Pi-agnostic: no pi runtime types. Pure src/ module.
+ */
+import { isVagueQuery } from "./queryReformulation/vagueness.js";
+import { extractExpansionTerms } from "./queryReformulation/tfidf.js";
+import { reciprocalRankFusion } from "./queryReformulation/rrf.js";
+import { cacheGet, cacheSet, maybeLogCacheStats, computeUncalibrated, QUERY_REFORM_CACHE_TTL_SECONDS, EXPANSION_NEIGHBOR_COUNT, EXPANSION_TOP_TERMS, VAGUE_MIN_WORDS, VAGUE_VERY_SHORT_WORDS, RRF_K, } from "./queryReformulation/cache.js";
+export { isVagueQuery } from "./queryReformulation/vagueness.js";
+export { extractExpansionTerms } from "./queryReformulation/tfidf.js";
+export { reciprocalRankFusion } from "./queryReformulation/rrf.js";
+export { readCacheStats } from "./queryReformulation/cache.js";
+// ---------------------------------------------------------------------------
+// Embedding Neighbor Finder (S43A-5)
+// ---------------------------------------------------------------------------
+/**
+ * Find the top-N nearest neighbor chunks by real cosine similarity over the
+ * real `context_chunks` store. Uses the provided embedder + scan function.
+ */
+export function findExpansionNeighbors(query, embedder, scan, neighborCount) {
+    const embedding = embedder.embed(query);
+    if (embedding.length === 0)
+        return [];
+    return scan(embedding, neighborCount);
+}
+// ---------------------------------------------------------------------------
+// Build Expanded Query
+// ---------------------------------------------------------------------------
+/**
+ * Build an expanded query by appending the top IDF-weighted terms from the
+ * expansion neighbors to the original query. Simple concatenation: the
+ * expanded string is `"originalQuery topTerm1 topTerm2 ..."`.
+ */
+export function buildExpandedQuery(original, terms, maxTerms = EXPANSION_TOP_TERMS) {
+    if (terms.length === 0)
+        return original;
+    const extra = terms.slice(0, maxTerms).map((t) => t.term).join(" ");
+    return `${original} ${extra}`;
+}
+// ---------------------------------------------------------------------------
+// Public API: reformulateQuery (S43A-9)
+// ---------------------------------------------------------------------------
+/**
+ * Reformulate a query using the fully local pipeline:
+ *   1. Check cache (TTL-gated)
+ *   2. Check vagueness — skip if not vague
+ *   3. Find embedding neighbors
+ *   4. Extract TF-IDF terms from neighbor texts
+ *   5. Build expanded query
+ *   6. Optionally re-embed and search (caller handles RRF fusion)
+ *
+ * Error contract:
+ *   - If the embedder throws OR the scan fails: the error propagates to the
+ *     caller (NOT caught here). The caller is responsible for the
+ *     `query_reformulation_failed` event + fallback to raw query.
+ *   - If the corpus is empty: returns raw query with rrfApplied: false and
+ *     skipReason: "empty_corpus".
+ *   - If the corpus is too thin (< 2 neighbors): returns raw query with
+ *     rrfApplied: false and skipReason: "thin_corpus".
+ *   - If the query is not vague: returns raw query with rrfApplied: false and
+ *     skipReason: "not_vague".
+ */
+export function reformulateQuery(query, embedder, scan, corpus, neighborTexts, opts = {}, log) {
+    const config = {
+        neighborCount: opts.neighborCount ?? EXPANSION_NEIGHBOR_COUNT,
+        topTerms: opts.topTerms ?? EXPANSION_TOP_TERMS,
+        vagueMinWords: opts.vagueMinWords ?? VAGUE_MIN_WORDS,
+        vagueVeryShortWords: opts.vagueVeryShortWords ?? VAGUE_VERY_SHORT_WORDS,
+        cacheTtlSeconds: opts.cacheTtlSeconds ?? QUERY_REFORM_CACHE_TTL_SECONDS,
+        rrfK: opts.rrfK ?? RRF_K,
+        searchLimit: opts.searchLimit ?? 5,
+    };
+    const ttlMs = config.cacheTtlSeconds * 1000;
+    // 0. Check cache
+    const cached = cacheGet(query, ttlMs);
+    if (cached) {
+        maybeLogCacheStats(log);
+        return {
+            result: cached,
+            rawNeighbors: cached.neighbors,
+            rawTerms: cached.neighbors.map((n) => ({ id: n.id, text: "" })),
+        };
+    }
+    // 1. Vagueness check
+    if (!isVagueQuery(query, { vagueMinWords: config.vagueMinWords, vagueVeryShortWords: config.vagueVeryShortWords })) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors: [],
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "not_vague",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "not_vague" });
+        return { result, rawNeighbors: [], rawTerms: [] };
+    }
+    // 2. Corpus check
+    if (corpus.totalDocs <= 0) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors: [],
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "empty_corpus",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "empty_corpus" });
+        return { result, rawNeighbors: [], rawTerms: [] };
+    }
+    // 3. Find embedding neighbors
+    const neighbors = findExpansionNeighbors(query, embedder, scan, config.neighborCount);
+    if (neighbors.length < 2) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors,
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "thin_corpus",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "thin_corpus" });
+        return { result, rawNeighbors: neighbors, rawTerms: [] };
+    }
+    // 4. Fetch neighbor texts for TF-IDF
+    const neighborIds = neighbors.map((n) => n.id);
+    const rawTerms = neighborTexts(neighborIds);
+    // 5. Extract expansion terms via TF-IDF
+    const expansionTerms = extractExpansionTerms(rawTerms, corpus, config.topTerms);
+    if (expansionTerms.length === 0) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors,
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "thin_corpus",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "thin_corpus" });
+        return { result, rawNeighbors: neighbors, rawTerms };
+    }
+    // 6. Build expanded query
+    const expanded = buildExpandedQuery(query, expansionTerms, config.topTerms);
+    const uncalibrated = computeUncalibrated(config);
+    const result = {
+        expanded,
+        original: query,
+        neighbors: neighbors.map((n) => ({ id: n.id, score: n.score })),
+        terms: expansionTerms,
+        rrfApplied: true,
+        fromCache: false,
+        uncalibrated,
+    };
+    cacheSet(query, result);
+    maybeLogCacheStats(log);
+    log?.info("query_reformulation", {
+        query: query.slice(0, 80),
+        expanded: expanded.slice(0, 120),
+        neighborCount: neighbors.length,
+        topTerms: expansionTerms.slice(0, 5).map((t) => t.term),
+        rrfApplied: true,
+        fromCache: false,
+        uncalibrated,
+    });
+    return { result, rawNeighbors: neighbors, rawTerms };
+}
+/**
+ * Full reformulation search pipeline:
+ *   1. Reformulate the query (embedding neighbors -> TF-IDF -> expansion).
+ *   2. If RRF was applied: run BOTH raw + expanded queries and fuse via RRF.
+ *   3. If no RRF: return raw-query results only.
+ */
+export function reformulationSearch(query, embedder, scan, corpus, neighborTexts, search, opts = {}, log) {
+    const config = {
+        neighborCount: opts.neighborCount ?? EXPANSION_NEIGHBOR_COUNT,
+        topTerms: opts.topTerms ?? EXPANSION_TOP_TERMS,
+        vagueMinWords: opts.vagueMinWords ?? VAGUE_MIN_WORDS,
+        vagueVeryShortWords: opts.vagueVeryShortWords ?? VAGUE_VERY_SHORT_WORDS,
+        cacheTtlSeconds: opts.cacheTtlSeconds ?? QUERY_REFORM_CACHE_TTL_SECONDS,
+        rrfK: opts.rrfK ?? RRF_K,
+        searchLimit: opts.searchLimit ?? 5,
+    };
+    const { result: reformulation } = reformulateQuery(query, embedder, scan, corpus, neighborTexts, opts, log);
+    // Always run the raw query search (needed for the S41 quality gate and as
+    // a fallback when RRF is not applied).
+    const rawSearchResults = search(query, config.searchLimit);
+    if (!reformulation.rrfApplied) {
+        return { fused: rawSearchResults, reformulation, rawSearchResults };
+    }
+    // RRF path: run expanded query search and fuse
+    const expandedResults = search(reformulation.expanded, config.searchLimit);
+    const rawRanked = rawSearchResults.map((r, i) => ({ id: r.id, rank: i + 1 }));
+    const expandedRanked = expandedResults.map((r, i) => ({ id: r.id, rank: i + 1 }));
+    const fused = reciprocalRankFusion([rawRanked, expandedRanked], config.rrfK);
+    return { fused, reformulation, rawSearchResults };
+}

package/dist/src/recallMetrics.js

@@ -0,0 +1,258 @@
+/**
+ * recallMetrics.ts — S45 CRAG quality metrics for recall evaluation.
+ *
+ * Four metrics evaluate the quality of a set of recall hits:
+ *   - relevance:     average cosine similarity (SearchHit.score) across hits
+ *   - coverage:      fraction of query sub-terms found in at least one hit summary
+ *   - diversity:     average pairwise cosine distance among top-K embeddings (MMR-style)
+ *   - specificity:   inverse document-frequency proxy via average chunk length
+ *
+ * Pi-agnostic: no pi runtime types, no network calls (PREVENT-PI-004).
+ * All config is passed explicitly; defaults are documented but NOT hardcoded
+ * inside the metric functions (the four constants from S45A.5 are also received
+ * via config, not baked in).
+ *
+ * Best-effort: all functions handle degenerate inputs (empty hits, missing
+ * embeddings, zero query terms) and return sensible defaults instead of NaN.
+ */
+import { cosineSimilarity } from "./embedder.js";
+import { STOP_WORDS } from "./config/stopwords.js";
+/** Default config — all values marked "uncalibrated" unless CRAG_CALIBRATED=true. */
+export const DEFAULT_RECALL_QUALITY_CONFIG = {
+    minDiversity: 0.3,
+    minCoverage: 0.4,
+    minRelevance: 0.5,
+    minOverallScore: 0.4,
+    weights: { relevance: 0.35, coverage: 0.25, diversity: 0.25, specificity: 0.15 },
+    specificityDivisor: 300,
+    specificityOptimalMin: 100,
+    specificityOptimalMax: 500,
+    idfBroadenRatio: 0.5,
+    stopwords: STOP_WORDS,
+    calibrated: false,
+};
+// ---------------------------------------------------------------------------
+// Metric helpers (exported individually for unit testing)
+// ---------------------------------------------------------------------------
+/**
+ * S45A.2: Relevance — average of SearchHit.score across all hits.
+ * Scores are already cosine similarities in [0, 1].
+ * Empty hits → 0.
+ */
+export function computeRelevance(hits) {
+    if (hits.length === 0)
+        return 0;
+    let sum = 0;
+    for (const h of hits)
+        sum += h.score;
+    return sum / hits.length;
+}
+/**
+ * S45A.3: Coverage — fraction of query subterms present in at least one
+ * hit's `checkpoint.summary`. Tokenization removes stopwords and short terms.
+ * Zero query terms after filtering → coverage = 1.0 (trivially satisfied).
+ */
+export function computeCoverage(query, hits, stopwords = STOP_WORDS) {
+    const queryTerms = tokenizeTerms(query, stopwords);
+    if (queryTerms.length === 0)
+        return 1.0;
+    if (hits.length === 0)
+        return 0;
+    let foundCount = 0;
+    for (const qt of queryTerms) {
+        const found = hits.some((h) => {
+            const summary = h.checkpoint.summary ?? "";
+            return summary.toLowerCase().includes(qt);
+        });
+        if (found)
+            foundCount++;
+    }
+    return foundCount / queryTerms.length;
+}
+/**
+ * S45A.4: Diversity — average pairwise cosine distance (1 - similarity)
+ * among hit embeddings. Filtered to hits with non-empty embedding arrays.
+ *
+ * Edge cases:
+ *   - < 2 valid hits → 1.0 (nothing to compare)
+ *   - empty-embedding checkpoints are skipped (no NaN path)
+ */
+export function computeDiversity(hits) {
+    const valid = hits.filter((h) => Array.isArray(h.checkpoint.embedding) && h.checkpoint.embedding.length > 0);
+    if (valid.length < 2)
+        return 1.0;
+    let totalDist = 0;
+    let pairCount = 0;
+    for (let i = 0; i < valid.length; i++) {
+        for (let j = i + 1; j < valid.length; j++) {
+            const sim = cosineSimilarity(valid[i].checkpoint.embedding, valid[j].checkpoint.embedding);
+            // cosineSimilarity returns 0 for empty vectors, but we've filtered those out.
+            // NaN/Infinity could theoretically arise from degenerate floats; guard defensively.
+            const dist = Number.isFinite(sim) ? 1 - sim : 1.0;
+            totalDist += dist;
+            pairCount++;
+        }
+    }
+    return pairCount > 0 ? totalDist / pairCount : 1.0;
+}
+/**
+ * S45A.5: Specificity — average chunk length (tokenEstimate) as a proxy
+ * for information density.
+ *
+ * Formula (all constants from config, NOT hardcoded):
+ *   avg = mean(tokenEstimate across hits; hits missing tokenEstimate → 0)
+ *   If avg < optimalMin / 2 → 0 (too short to be specific)
+ *   base = avg / divisor
+ *   If avg > optimalMax:
+ *     ramp = 1 - (avg - optimalMax) / optimalMax  // linear to 0 at 2*optimalMax
+ *     capped = max(0, ramp)
+ *     specificity = min(base, capped)
+ *   else:
+ *     specificity = min(base, 1)
+ *
+ * Edge cases: empty hits → 1.0 (nothing to penalize).
+ */
+export function computeSpecificity(hits, config) {
+    if (hits.length === 0)
+        return 1.0;
+    const { specificityDivisor: divisor, specificityOptimalMin: optMin, specificityOptimalMax: optMax } = config;
+    let sum = 0;
+    for (const h of hits) {
+        sum += h.checkpoint.tokenEstimate ?? 0;
+    }
+    const avg = sum / hits.length;
+    // Below optMin/2 → too short to be specific
+    if (avg < optMin / 2)
+        return 0;
+    const base = divisor > 0 ? avg / divisor : 0;
+    const clamped = base > 1 ? 1 : base;
+    if (avg > optMax) {
+        // Linear ramp-down from optMax to 0 at 2*optMax
+        const ramp = optMax > 0 ? 1 - (avg - optMax) / optMax : 0;
+        const capped = ramp > 0 ? ramp : 0;
+        return clamped < capped ? clamped : capped;
+    }
+    return clamped;
+}
+// ---------------------------------------------------------------------------
+// Composite evaluation
+// ---------------------------------------------------------------------------
+/**
+ * Normalize a four-weight vector so its components sum to 1.0.
+ * If all weights are 0, return equal weights (0.25 each).
+ * Used defensively since users may set non-sum-1 weights via env.
+ */
+export function normalizeWeights(w) {
+    const sum = w.relevance + w.coverage + w.diversity + w.specificity;
+    if (sum === 0)
+        return { relevance: 0.25, coverage: 0.25, diversity: 0.25, specificity: 0.25 };
+    return {
+        relevance: w.relevance / sum,
+        coverage: w.coverage / sum,
+        diversity: w.diversity / sum,
+        specificity: w.specificity / sum,
+    };
+}
+/**
+ * S45A.6: Evaluate recall quality — compute all four metrics and return a
+ * composite result with pass/fail + recommendation.
+ */
+export function evaluateRecall(query, hits, config) {
+    const weights = normalizeWeights(config.weights);
+    const breakdown = {
+        relevance: computeRelevance(hits),
+        coverage: computeCoverage(query, hits, config.stopwords),
+        diversity: computeDiversity(hits),
+        specificity: computeSpecificity(hits, config),
+    };
+    const score = weights.relevance * breakdown.relevance +
+        weights.coverage * breakdown.coverage +
+        weights.diversity * breakdown.diversity +
+        weights.specificity * breakdown.specificity;
+    const pass = score >= config.minOverallScore &&
+        breakdown.relevance >= config.minRelevance &&
+        breakdown.coverage >= config.minCoverage &&
+        breakdown.diversity >= config.minDiversity;
+    const recommendation = buildRecommendation(breakdown, score, config);
+    return {
+        pass,
+        score,
+        breakdown,
+        recommendation,
+        uncalibrated: !config.calibrated,
+        weights,
+    };
+}
+// ---------------------------------------------------------------------------
+// Recommendation builder
+// ---------------------------------------------------------------------------
+/**
+ * Build a human-readable recommendation based on which metric is lowest.
+ * Uses env-overridable thresholds from config (not hardcoded).
+ */
+function buildRecommendation(breakdown, score, config) {
+    // Check each metric against its threshold (lowest first for specificity).
+    const issues = [];
+    if (breakdown.diversity < config.minDiversity) {
+        issues.push("chunks are too similar — consider expanding K or using MMR");
+    }
+    if (breakdown.coverage < config.minCoverage) {
+        issues.push("query terms not well covered — consider query expansion");
+    }
+    if (breakdown.relevance < config.minRelevance) {
+        issues.push("chunks may be irrelevant — consider stricter threshold");
+    }
+    if (score < config.minOverallScore) {
+        issues.push("quality is low — recommend re-retrieval with expanded query");
+    }
+    return issues.length > 0 ? issues.join("; ") : null;
+}
+/**
+ * Compute recall quality metrics for a set of search hits.
+ *
+ * Pi-agnostic: takes query text and SearchHit results directly, no pi types.
+ * The `config` parameter is optional — when omitted, DEFAULT_RECALL_QUALITY_CONFIG
+ * is used (labelled uncalibrated).
+ *
+ * This is the primary export for external callers (e.g., recall.ts integration).
+ */
+export function computeRecallMetrics(query, results, config) {
+    const effectiveConfig = {
+        ...DEFAULT_RECALL_QUALITY_CONFIG,
+        ...config,
+        // Merge weights shallowly if both sides exist
+        weights: {
+            ...DEFAULT_RECALL_QUALITY_CONFIG.weights,
+            ...(config?.weights ?? {}),
+        },
+    };
+    const result = evaluateRecall(query, results, effectiveConfig);
+    return {
+        breakdown: result.breakdown,
+        score: result.score,
+        pass: result.pass,
+        recommendation: result.recommendation,
+        uncalibrated: result.uncalibrated,
+    };
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/**
+ * Tokenize text into lowercased alphanumeric terms, filtering stop words
+ * and terms shorter than 3 characters.
+ */
+function tokenizeTerms(text, stopwords) {
+    return text
+        .toLowerCase()
+        .split(/[^a-z0-9]+/)
+        .filter((t) => t.length >= 3 && !stopwords.has(t));
+}
+/**
+ * Estimate block tokens for a piece of text (rough approximation).
+ * Used by the specificity metric fallback when tokenEstimate is missing.
+ */
+export function estimateBlockTokens(text) {
+    // ~4 chars per token for English text
+    return Math.ceil(text.length / 4);
+}