syncstaff-mcp 0.2.3

package/dist/lib/index/lexical.js

@@ -0,0 +1,462 @@
+import { byCodeUnit } from "./order.js";
+const DEFAULT_STOP_WORDS = new Set("a an and are as at be by for from in is it of on or the to with".split(" "));
+const tokenise = (value) => value
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .toLowerCase()
+    .match(/[A-Za-z0-9_]+/g)
+    ?.flatMap((part) => part.split("_"))
+    .filter(Boolean) ?? [];
+const fields = (page) => [page.title, page.content, page.summary, page.target_path, page.body ?? "", page.lexical_vocabulary ?? ""].join(" ");
+/** Parse the candidate-only vocabulary weight; malformed values fail closed. */
+export function vocabularyWeightFromEnv(raw = process.env.KEEL_RETRIEVAL_VOCABULARY) {
+    if (!raw?.trim())
+        return 0;
+    const value = Number(raw.trim());
+    return Number.isFinite(value) && value > 0 && value <= 3 ? value : 0;
+}
+/** Length-normalisation strength per field, the usual BM25 `b`. */
+const FIELD_B = 0.75;
+/**
+ * All ones: exactly today's behaviour, and that is the point.
+ *
+ * Under this weighting every field is counted once, which is what
+ * `fields().join(" ")` did. The ablation baseline is therefore the same code
+ * path as the treatment rather than a different one, so a measured difference
+ * cannot be an artefact of which branch ran.
+ */
+export const IDENTITY_FIELD_WEIGHTS = { path: 1, facts: 1, summary: 1, body: 1, norm: "global" };
+/**
+ * Measured 28 Aug 2026 over four epochs, and the identity weighting is staying
+ * the default. Recorded here because the shape of the failure is not obvious.
+ *
+ *   path=4,facts=3 with global normalisation
+ *     recall@10 +0.0069 holdout, cluster CI [+0.0023, +0.0120]
+ *     read tokens +5% to +41% per instance, CI clearing zero on three epochs
+ *
+ *   path=4,facts=3 with textbook per-field normalisation
+ *     recall@10 +0.0000 holdout, cluster CI [-0.0082, +0.0077]
+ *     read tokens +8.9% train, cluster CI [+1.5%, +14.6%]
+ *
+ * The first looks like a small win. It is very likely a size artefact: adding
+ * `weight x len(field)` to one document length penalises whichever documents
+ * those short fields are a large share of — the small files — so up-weighting
+ * `path` quietly promoted big ones. The second removes exactly that bias, and
+ * the recall gain disappears with it while the token cost does not.
+ *
+ * A degenerate baseline (rule 2) makes the confounder concrete. Serving the ten
+ * largest files in the corpus, ignoring the query completely, scores recall@10
+ * of 0.1596 on keel200 and 0.0793 on pw1200 — far below real retrieval, but
+ * three to twenty-five times a same-sized random pick. File size genuinely
+ * correlates with being edited, so any mechanism whose gain arrives together
+ * with a file-size increase has to be assumed guilty until separated.
+ *
+ * So the parent hypothesis — that distinguishing fields would let identifiers
+ * and paths outweigh incidental body text — is not supported on its own. The
+ * machinery stays because it is how the *other* half of that item (a richer
+ * extracted vocabulary) would be tested, and that is a different mechanism:
+ * new terms rather than reweighted ones.
+ */
+/** Fixed iteration order, so an index build is deterministic across runs. */
+const FIELD_ORDER = ["path", "facts", "summary", "body"];
+/**
+ * Parse `path=4,facts=3,summary=1,body=1` from the environment.
+ *
+ * One build serves every weighting, so a sweep cannot be confounded with
+ * anything else that changed between two builds — the same reason `KEEL_CHUNK`
+ * and `KEEL_NO_GRAPH` are switches rather than branches.
+ *
+ * Anything unparseable falls back to the identity weighting rather than
+ * throwing. A malformed weight string in a benchmark harness should produce the
+ * documented default, not abort a run that has already paid for its index.
+ */
+export function fieldWeightsFromEnv(raw = process.env.KEEL_FIELD_WEIGHTS ?? process.env.CHARTER_FIELD_WEIGHTS) {
+    if (!raw)
+        return IDENTITY_FIELD_WEIGHTS;
+    const weights = { ...IDENTITY_FIELD_WEIGHTS };
+    for (const part of raw.split(",")) {
+        const [name, value] = part.split("=").map((piece) => piece.trim());
+        if (name === "norm") {
+            if (value === "field" || value === "global")
+                weights.norm = value;
+            continue;
+        }
+        const parsed = Number(value);
+        if (!FIELD_ORDER.includes(name) || !Number.isInteger(parsed) || parsed < 0)
+            continue;
+        weights[name] = parsed;
+    }
+    return weights;
+}
+/** A page split into the fields the ranker weighs separately. */
+const fieldTexts = (page) => ({
+    // Title and target_path are one field, not two. A file page's title *is* its
+    // path, so counting them separately would silently double the path's weight
+    // and make the configured number mean something other than it says.
+    path: `${page.title} ${page.target_path}`,
+    facts: page.content,
+    summary: page.summary,
+    body: page.body ?? "",
+});
+const signalOrder = ["phrase", "path", "identifier", "action", "term"];
+const signalWeight = { phrase: 3, path: 2.5, identifier: 2.2, action: 0.8, term: 1 };
+const actions = new Set(["add", "align", "change", "delete", "fix", "implement", "move", "refactor", "remove", "rename", "simplify", "update"]);
+/** Split an issue into independent deterministic search signals. */
+export function decomposeQuery(query) {
+    const signals = [];
+    const seen = new Set();
+    const add = (kind, raw) => {
+        const value = raw.trim();
+        if (!value)
+            return;
+        const key = `${kind}\0${value.toLowerCase()}`;
+        if (seen.has(key))
+            return;
+        seen.add(key);
+        signals.push({ kind, query: value, weight: signalWeight[kind] });
+    };
+    for (const match of query.matchAll(/["“”']([^"“”']{2,})["“”']/g))
+        add("phrase", match[1]);
+    const tokens = query.match(/[A-Za-z0-9_./-]+/g) ?? [];
+    for (const token of tokens) {
+        if (token.includes("/") || /\.[A-Za-z0-9_-]+$/.test(token))
+            add("path", token);
+        else if (actions.has(token.toLowerCase()))
+            add("action", token.toLowerCase());
+        else if (/[A-Z]/.test(token) || token.includes("_") || /\.[A-Za-z_$]/.test(token))
+            add("identifier", token);
+    }
+    const terms = [...new Set(tokenise(query).filter((term) => term.length >= 3 && !DEFAULT_STOP_WORDS.has(term)))];
+    if (terms.length)
+        add("term", terms.join(" "));
+    if (!signals.length && query.trim())
+        add("term", query);
+    return signals.sort((a, b) => signalOrder.indexOf(a.kind) - signalOrder.indexOf(b.kind) || byCodeUnit(a.query, b.query));
+}
+const strongIdentifier = (token) => token.includes("_") || token.includes(".") || /[a-z][A-Z]/.test(token) || /^_/.test(token);
+const uniqueQueries = (queries, max = 4) => {
+    const seen = new Set();
+    const output = [];
+    for (const query of queries) {
+        const value = query.trim();
+        const key = value.toLowerCase();
+        if (!value || seen.has(key))
+            continue;
+        seen.add(key);
+        output.push(value);
+        if (output.length >= max)
+            break;
+    }
+    return output;
+};
+/**
+ * Route one issue into leg-specific query variants.
+ *
+ * Paths and code identifiers are reliable lexical/symbol anchors but are
+ * poor semantic text. Actions describe the requested change, not the file's
+ * subject. The semantic leg therefore receives the raw question plus a
+ * bounded concept variant with path/identifier/action noise removed. Keeping
+ * the raw question first preserves the original ranking as the primary
+ * semantic signal while the variants recover paraphrased concepts.
+ */
+export function routeQuery(query) {
+    const rawTokens = query.match(/[A-Za-z0-9_./-]+/g) ?? [];
+    const pathTokens = rawTokens.filter((token) => token.includes("/") || /\.[A-Za-z0-9_-]+$/.test(token));
+    const identifierTokens = rawTokens.filter((token) => !pathTokens.includes(token) && strongIdentifier(token));
+    const pathTerms = new Set(pathTokens.flatMap((token) => tokenise(token)));
+    const identifierTerms = new Set(identifierTokens.flatMap((token) => tokenise(token)));
+    const concepts = [...new Set(tokenise(query).filter((term) => term.length >= 3 &&
+            !DEFAULT_STOP_WORDS.has(term) &&
+            !actions.has(term) &&
+            !pathTerms.has(term) &&
+            !identifierTerms.has(term)))];
+    const conceptQuery = concepts.join(" ");
+    const phrases = decomposeQuery(query).filter((signal) => signal.kind === "phrase").map((signal) => signal.query);
+    const symbolQueries = uniqueQueries(identifierTokens.length ? identifierTokens : [query]);
+    const semanticQueries = uniqueQueries([query, ...phrases, conceptQuery]);
+    return {
+        raw_query: query,
+        lexical_query: query,
+        symbol_queries: symbolQueries,
+        semantic_queries: semanticQueries,
+        path_queries: uniqueQueries(pathTokens),
+        concept_queries: conceptQuery ? [conceptQuery] : [],
+    };
+}
+/** A reusable, dependency-free BM25 index for a stable page corpus. */
+export class LexicalIndex {
+    pages;
+    postings;
+    vocabularyPostings;
+    vocabulary;
+    lengths;
+    vocabularyLengths;
+    averageLength;
+    averageVocabularyLength;
+    weights;
+    fieldNorm;
+    /**
+     * Mean token count of each field across the corpus.
+     *
+     * Only the `field` normalisation mode needs these, and only as denominators,
+     * so a field nobody weighs is never tokenised.
+     */
+    averageFieldLengths(pages) {
+        const totals = FIELD_ORDER.map(() => 0);
+        for (const page of pages) {
+            const texts = fieldTexts(page);
+            FIELD_ORDER.forEach((field, slot) => {
+                if (this.weights[field] > 0)
+                    totals[slot] += tokenise(texts[field]).length;
+            });
+        }
+        return totals.map((total) => total / Math.max(1, pages.length));
+    }
+    constructor(allPages, weights = fieldWeightsFromEnv()) {
+        this.weights = weights;
+        this.fieldNorm = weights.norm ?? "global";
+        // Chunks exist so the vector leg can read a whole file; they carry their
+        // file's `target_path`, so indexing them here would enter every oversized
+        // file several times over. That corrupts document frequency and average
+        // length for the entire corpus, and it puts duplicate paths in a response
+        // — which is how this was found, as a recall@10 of 1.03 against a gold set
+        // where 1.0 is the ceiling.
+        //
+        // Enforced here rather than at the call site because `retrieve()` accepts a
+        // prebuilt index, so a filter upstream protects only the callers that do
+        // not supply one. The benchmark harness supplied one.
+        const pages = allPages.filter((page) => page.page_type !== "chunk");
+        this.pages = pages;
+        const builders = new Map();
+        const vocabularyBuilders = new Map();
+        this.lengths = [];
+        this.vocabularyLengths = [];
+        let totalLength = 0;
+        let totalVocabularyLength = 0;
+        // Per-field average lengths, needed only by the "field" mode — so the
+        // default path stays a single pass over the corpus. `body` is the file's
+        // whole text and tokenising is the expensive part of an index build; making
+        // every caller pay twice for a mode they are not using would be a real
+        // regression for an experiment.
+        const fieldAverages = this.fieldNorm === "field" ? this.averageFieldLengths(pages) : null;
+        for (let index = 0; index < pages.length; index += 1) {
+            const texts = fieldTexts(pages[index]);
+            const counts = new Map();
+            let length = 0;
+            FIELD_ORDER.forEach((field, slot) => {
+                const weight = this.weights[field];
+                if (weight <= 0)
+                    return;
+                const tokens = tokenise(texts[field]);
+                length += tokens.length * weight;
+                // Two ways to combine fields, and the difference is the whole reason
+                // the first attempt at this cost tokens instead of saving them.
+                //
+                // "global" is the original behaviour: sum the weighted frequencies,
+                // sum the weighted lengths, and normalise once at scoring time against
+                // one corpus-wide average. Under identity weights that is exactly the
+                // concatenated bag this class has always used, which is what makes it
+                // an honest ablation baseline.
+                //
+                // It is also subtly wrong as soon as the weights are not all one.
+                // Adding `weight x len(field)` to a single document length penalises
+                // the documents those short fields are a large *share* of — the small
+                // files. A large file's length is dominated by its body, so weighting
+                // `path` up barely moves its normalised length, while a small file's
+                // grows proportionally. Measured over four epochs: recall rose
+                // (+0.0069 holdout) and read tokens rose with it, +5% to +41% per
+                // instance, because the ranking had quietly begun preferring larger
+                // files. That is the chunking trade again — real recall bought with
+                // tokens — and here it is an artefact of the formulation, not of the
+                // idea being tested.
+                //
+                // "field" is textbook BM25F: normalise each field against its own
+                // average length *before* weighting, so a field is only ever compared
+                // with the same field in other documents and a short path is not
+                // evidence that the document is short. Saturation then applies once to
+                // the combined value, which is why scoring skips its length term in
+                // this mode.
+                const factor = fieldAverages
+                    ? weight / (1 - FIELD_B + FIELD_B * (tokens.length / Math.max(1, fieldAverages[slot])))
+                    : weight;
+                for (const token of tokens)
+                    counts.set(token, (counts.get(token) ?? 0) + factor);
+            });
+            this.lengths.push(length);
+            totalLength += length;
+            const vocabularyCounts = new Map();
+            for (const token of tokenise(pages[index].lexical_vocabulary ?? "")) {
+                vocabularyCounts.set(token, (vocabularyCounts.get(token) ?? 0) + 1);
+            }
+            const vocabularyLength = [...vocabularyCounts.values()].reduce((total, count) => total + count, 0);
+            this.vocabularyLengths.push(vocabularyLength);
+            totalVocabularyLength += vocabularyLength;
+            for (const [term, count] of vocabularyCounts) {
+                const posting = vocabularyBuilders.get(term) ?? [];
+                posting.push(index, count);
+                vocabularyBuilders.set(term, posting);
+            }
+            for (const [term, count] of counts) {
+                const posting = builders.get(term) ?? [];
+                // Flat [document index, term frequency] pairs are materially smaller
+                // than one object per posting on an 80k-file corpus. Float32 rather
+                // than Uint32 because field normalisation produces fractional
+                // frequencies; it is the same four bytes per element, and a document
+                // index stays exactly representable well past any corpus this indexes.
+                posting.push(index, count);
+                builders.set(term, posting);
+            }
+        }
+        this.postings = new Map([...builders].map(([term, posting]) => [term, Float32Array.from(posting)]));
+        this.vocabularyPostings = new Map([...vocabularyBuilders].map(([term, posting]) => [term, Float32Array.from(posting)]));
+        this.vocabulary = [...new Set([...this.postings.keys(), ...this.vocabularyPostings.keys()])].sort(byCodeUnit);
+        this.averageLength = totalLength / Math.max(1, pages.length);
+        this.averageVocabularyLength = totalVocabularyLength / Math.max(1, pages.length);
+    }
+    documentFrequency(term, includeVocabulary = false) {
+        const posting = this.postings.get(term);
+        if (!includeVocabulary)
+            return (posting?.length ?? 0) / 2;
+        const documents = new Set();
+        for (const current of [posting, this.vocabularyPostings.get(term)]) {
+            for (let i = 0; i < (current?.length ?? 0); i += 2)
+                documents.add(current[i]);
+        }
+        return documents.size;
+    }
+    matchingPostings(term, vocabularyWeight = 0) {
+        const matches = new Map();
+        const add = (posting, multiplier = 1) => {
+            for (let i = 0; i < posting.length; i += 2) {
+                const document = posting[i];
+                matches.set(document, (matches.get(document) ?? 0) + posting[i + 1] * multiplier);
+            }
+        };
+        const addMatching = (termToMatch) => {
+            const exact = this.postings.get(termToMatch);
+            if (exact)
+                add(exact);
+            if (vocabularyWeight > 0) {
+                const structural = this.vocabularyPostings.get(termToMatch);
+                if (structural)
+                    add(structural, vocabularyWeight);
+            }
+        };
+        if (term.length < 4) {
+            addMatching(term);
+            return matches;
+        }
+        // Vocabulary is sorted by code unit, so the first lower-bound match and
+        // a contiguous prefix scan replace a full page scan while preserving the
+        // original `token.startsWith(term)` behavior.
+        let low = 0;
+        let high = this.vocabulary.length;
+        while (low < high) {
+            const middle = (low + high) >>> 1;
+            if (byCodeUnit(this.vocabulary[middle], term) < 0)
+                low = middle + 1;
+            else
+                high = middle;
+        }
+        for (let i = low; i < this.vocabulary.length && this.vocabulary[i].startsWith(term); i += 1) {
+            addMatching(this.vocabulary[i]);
+        }
+        return matches;
+    }
+    search(query, options = {}) {
+        const stopWords = options.stopWords ?? DEFAULT_STOP_WORDS;
+        const limit = options.limit ?? 10;
+        const ceiling = options.documentFrequencyCeiling ?? 0.2;
+        const vocabularyWeight = Number.isFinite(options.vocabularyWeight) && (options.vocabularyWeight ?? 0) > 0
+            ? Math.min(3, options.vocabularyWeight)
+            : 0;
+        const includeVocabulary = vocabularyWeight > 0;
+        const rawTerms = tokenise(query).filter((term) => !stopWords.has(term));
+        const rankedTerms = [...new Set(rawTerms)]
+            .sort((a, b) => this.documentFrequency(a, includeVocabulary) - this.documentFrequency(b, includeVocabulary) || byCodeUnit(a, b));
+        const rareTerms = rankedTerms.filter((term) => this.documentFrequency(term, includeVocabulary) / Math.max(1, this.pages.length) <= ceiling);
+        // `rankedTerms` is ascending by document frequency, so its head is the most
+        // discriminative part of the query. Always score it, even when the ceiling
+        // would reject it.
+        //
+        // The ceiling alone was safe only while pages were structural cards. Once a
+        // page carries the file's own text, an identifier appears both where it is
+        // defined and at every call site, so the very terms that locate a file are
+        // the ones whose document frequency climbs past the ceiling — and dropping
+        // them left the query matching on its vaguest words. BM25 already discounts
+        // common terms through IDF; discarding them outright double-counts that and
+        // loses the signal instead of weighting it.
+        const floor = rankedTerms.slice(0, Math.min(3, rankedTerms.length));
+        const terms = [...new Set([...rareTerms, ...floor])];
+        const score = (wanted) => {
+            const matchedByDocument = new Map();
+            for (const term of wanted) {
+                for (const [document, count] of this.matchingPostings(term, vocabularyWeight)) {
+                    const current = matchedByDocument.get(document) ?? { matched: [], counts: new Map() };
+                    current.matched.push(term);
+                    current.counts.set(term, count);
+                    matchedByDocument.set(document, current);
+                }
+            }
+            const hits = [];
+            for (const [document, match] of matchedByDocument) {
+                const k1 = 1.2;
+                const b = 0.75;
+                let total = 0;
+                for (const term of match.matched) {
+                    const count = match.counts.get(term) ?? 0;
+                    const documentFrequency = this.documentFrequency(term, includeVocabulary);
+                    const inverse = Math.log(1 + (this.pages.length - documentFrequency + 0.5) / (documentFrequency + 0.5));
+                    // In `field` mode the length term is already inside `count` — each
+                    // field was divided by its own average before being weighted and
+                    // summed — so applying a second, document-wide normalisation here
+                    // would normalise twice and undo the thing the mode exists to fix.
+                    const normalisation = this.fieldNorm === "field"
+                        ? 1
+                        : 1 - b + b * (this.lengths[document] + vocabularyWeight * this.vocabularyLengths[document]) /
+                            Math.max(1, this.averageLength + vocabularyWeight * this.averageVocabularyLength);
+                    total += inverse * ((count * (k1 + 1)) / (count + k1 * normalisation));
+                }
+                hits.push({ page: this.pages[document], score: total, matched_terms: match.matched });
+            }
+            return hits.sort((a, b) => b.score - a.score || byCodeUnit(a.page.page_id, b.page.page_id));
+        };
+        let hits = score(terms);
+        const widened = hits.length === 0 && terms.length !== rankedTerms.length;
+        if (widened)
+            hits = score(rankedTerms);
+        return { query, terms, widened, hits: hits.slice(0, limit) };
+    }
+    searchMulti(query, options = {}) {
+        const limit = options.limit ?? 10;
+        const signals = decomposeQuery(query);
+        const candidateLimit = Math.min(100, Math.max(limit * 3, 20));
+        const byPage = new Map();
+        for (const signal of signals) {
+            const result = this.search(signal.query, { ...options, limit: candidateLimit });
+            result.hits.forEach((hit, rank) => {
+                const current = byPage.get(hit.page.page_id) ?? { page: hit.page, score: 0, terms: new Set(), kinds: new Set() };
+                current.score += signal.weight / (rank + 1);
+                if (signal.kind === "phrase" && fields(hit.page).toLowerCase().includes(signal.query.toLowerCase()))
+                    current.score += 1;
+                hit.matched_terms.forEach((term) => current.terms.add(term));
+                current.kinds.add(signal.kind);
+                byPage.set(hit.page.page_id, current);
+            });
+        }
+        const hits = [...byPage.values()]
+            .map((hit) => ({ page: hit.page, score: hit.score, matched_terms: [...hit.terms].sort(byCodeUnit), signal_kinds: [...hit.kinds].sort((a, b) => signalOrder.indexOf(a) - signalOrder.indexOf(b)) }))
+            .sort((a, b) => b.score - a.score || b.signal_kinds.length - a.signal_kinds.length || byCodeUnit(a.page.page_id, b.page.page_id));
+        return { query, terms: signals.map((signal) => signal.query), widened: false, hits: hits.slice(0, limit) };
+    }
+}
+export function buildLexicalIndex(pages, weights) {
+    return new LexicalIndex(pages, weights);
+}
+/** Search retrieval pages with a dependency-free BM25-style ranker. */
+export function searchLexical(pages, query, options = {}) {
+    return buildLexicalIndex(pages).search(query, options);
+}
+/** Search a fresh lexical index using decomposed phrase/path/identifier signals. */
+export function searchLexicalMulti(pages, query, options = {}) {
+    return buildLexicalIndex(pages).searchMulti(query, options);
+}

package/dist/lib/index/order.js

@@ -0,0 +1,43 @@
+/**
+ * The one comparator the index is allowed to sort with.
+ *
+ * This module exists because of a bug that only appeared the first time the
+ * pipeline read a real repository. `file-index.ts` ordered files with the
+ * default `Array.prototype.sort`, while `incremental.ts` ordered them with
+ * `localeCompare`. Those are different collations: default sort compares UTF-16
+ * code units, so "README.md" precedes "package-lock.json" because 'R' is 82
+ * and 'p' is 112; `localeCompare` folds case and orders them the other way.
+ *
+ * The result was an incremental index that held exactly the same files with
+ * exactly the same fingerprint in a different order — which broke the
+ * byte-identity invariant that the whole incremental design rests on. Every
+ * synthetic test passed, because fixtures were named a.ts and b.ts and the two
+ * collations agree on those.
+ *
+ * The deeper hazard is worse than the mismatch. `localeCompare` is
+ * locale-dependent: its result can differ between two machines with different
+ * ICU data or a different default locale. A subsystem whose entire purpose is
+ * letting two machines agree on what they are looking at must never order
+ * anything by a rule that varies between machines. That it happened to produce
+ * a stable fingerprint here is luck — `indexFingerprint` sorts its records with
+ * the default comparator — not design.
+ *
+ * So: code-unit ordering, everywhere, by one function nobody has to remember
+ * to choose. `localeCompare` is banned from this directory, and
+ * local-order.test.ts enforces that with a grep.
+ */
+/**
+ * Compare two strings by UTF-16 code unit.
+ *
+ * Identical on every machine, in every locale, forever. Not the right
+ * comparator for showing a list to a human — "Zebra" sorts before "apple" —
+ * and that is an acceptable trade, because these orderings exist to make bytes
+ * reproducible rather than to be read.
+ */
+export function byCodeUnit(a, b) {
+    return a < b ? -1 : a > b ? 1 : 0;
+}
+/** Order objects by one string field, by code unit. */
+export function byField(select) {
+    return (a, b) => byCodeUnit(select(a), select(b));
+}