pikelet-wasm 0.7.0 → 0.8.1

package/complete/builder.d.ts

@@ -1,5 +1,5 @@
 /**
- * pikelet-wasm/complete/builder — Node-only assembly of complete .pancake
+ * pikelet-wasm/complete/builder — Node-only assembly of complete .pikelet
  * artifacts (spec/COMPLETE_PROFILE.md). The wire-format constants here are
  * the same objects the reader uses (complete/format.mjs).
  */
@@ -15,8 +15,8 @@ export declare function canonicalJson(value: unknown): string;
 export declare const align16: (n: number) => number;
 
 /** Manifest profile strings and the header formatVersion each one implies. */
-export declare const PROFILE_V1: 'pancake-complete-v1';
-export declare const PROFILE_V2: 'pancake-complete-v2';
+export declare const PROFILE_V1: 'pikelet-complete-v1';
+export declare const PROFILE_V2: 'pikelet-complete-v2';
 export declare const FORMAT_VERSIONS: Record<string, number>;
 /** Corpus layout name for format 2 (per-record digests behind a page table). */
 export declare const CORPUS_LAYOUT_V2: 'records-v2';
@@ -52,7 +52,7 @@ export declare function buildInlineTransformerEncoderSegment(input: {
  * buildCorpusSegment() corpus); mismatches throw. Segments with a kind this
  * spec does not name must carry an explicit kindNumber (readers skip them).
  */
-export declare function assemblePancakeFile(
+export declare function assemblePikeletFile(
   manifestFields: Record<string, unknown> & { profile: string; corpus: { records: number } & Record<string, unknown> },
   segments: Array<{ kind: string; bytes: Buffer | Uint8Array; kindNumber?: number }>,
   outPath: string

package/complete/builder.mjs

@@ -81,8 +81,25 @@ export function buildQueryInterpSegment(kind, encoderBytes, calibrationBytes) {
 // and skips a small function-word stopword list — readers need no
 // coordination with the list, because an unindexed query token simply finds
 // no postings.
+//
+// Interrogative words (how/does/what/who/why/where/when/which/do/did) are
+// deliberately in this list, not just the base one: without them, a
+// natural-language question's own scaffolding becomes real, indexable BM25
+// content — a passage containing literal dialogue like "How does Georgiana
+// get on, Darcy?" then wins a high lexical score against the query "How
+// does Elizabeth initially react to Mr. Darcy?" purely from "how does" and
+// "darcy" co-occurring, with zero topical relevance. Confirmed live: this
+// pushed a genuinely correct chunk (ranked #2 by the vector search alone)
+// completely out of the fused top-10, replaced by RRF-boosted noise. Not
+// borrowed wholesale from calibrate.mjs's COVERAGE_STOPWORDS, which also
+// excludes domain words ("tell", "explain", "overview", "history",
+// "definition") that are legitimate lexical CONTENT on many real corpora
+// (a docs site literally titled "History") — this list only adds the
+// unambiguous interrogative set, since the lexical index (unlike coverage)
+// also indexes passage text, not just the query.
 const LEXICAL_LAYOUT = 'bm25-v1';
-const LEXICAL_STOPWORDS = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'at', 'by', 'for', 'with', 'from', 'into', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'it', 'its', 'this', 'that', 'these', 'those', 'as', 'their', 'they', 'them', 'his', 'her', 'has', 'have', 'had', 'not', 'can', 'will', 'if', 'we', 'you', 'your', 'i']);
+const LEXICAL_STOPWORDS = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'at', 'by', 'for', 'with', 'from', 'into', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'it', 'its', 'this', 'that', 'these', 'those', 'as', 'their', 'they', 'them', 'his', 'her', 'has', 'have', 'had', 'not', 'can', 'will', 'if', 'we', 'you', 'your', 'i',
+  'how', 'does', 'did', 'do', 'what', 'who', 'why', 'where', 'when', 'which']);
 const lexTokenize = (text) => (String(text).toLowerCase().match(/[a-z0-9']+/g) || [])
   .filter((w) => w.length >= 2 && w.length <= 32 && !LEXICAL_STOPWORDS.has(w));
 
@@ -228,7 +245,7 @@ export function buildLexicalSegment(texts) {
   };
 }
 
-// Corpus layout v1 (format 1, profile pancake-complete-v1): count + offsets +
+// Corpus layout v1 (format 1, profile pikelet-complete-v1): count + offsets +
 // records, integrity by whole-segment digest only. Kept for producing
 // format-1 files (compatibility fixtures, readers that predate format 2);
 // new artifacts use buildCorpusSegment() below.
@@ -248,7 +265,7 @@ export function buildCorpusSegmentFromBuffers(records) {
   return out;
 }
 
-// Corpus layout v2 (format 2, profile pancake-complete-v2): per-record
+// Corpus layout v2 (format 2, profile pikelet-complete-v2): per-record
 // integrity that a reader can check on the single range read that hydrates
 // a record, without fetching the rest of the segment (contract 4.1 / 7):
 //
@@ -352,7 +369,7 @@ export async function measureRecommendedRerank({
     async read(offset, length) { return bytes.subarray(offset, offset + length); },
     async close() {},
   };
-  const sketch = await artifactModule.PancakeSketchArtifact.open(source);
+  const sketch = await artifactModule.PikeletSketchArtifact.open(source);
   try {
     const count = sketch.count;
     if (count === 0) throw new Error('cannot measure rerank on an empty sketch');
@@ -432,16 +449,16 @@ function selfQueriesFromSnapshot(graph, maxQueries) {
   return queries;
 }
 
-// Container format version follows the profile string: pancake-complete-v1
-// files carry corpus layout v1 and header version 1; pancake-complete-v2
+// Container format version follows the profile string: pikelet-complete-v1
+// files carry corpus layout v1 and header version 1; pikelet-complete-v2
 // files carry corpus layout records-v2 (buildCorpusSegment) and header
 // version 2, so readers that predate per-record integrity reject them
 // explicitly instead of misreading the corpus tables.
-export const PROFILE_V1 = 'pancake-complete-v1';
-export const PROFILE_V2 = 'pancake-complete-v2';
+export const PROFILE_V1 = 'pikelet-complete-v1';
+export const PROFILE_V2 = 'pikelet-complete-v2';
 export const FORMAT_VERSIONS = { [PROFILE_V1]: 1, [PROFILE_V2]: 2 };
 
-export function assemblePancakeFile(manifestFields, segments, outPath) {
+export function assemblePikeletFile(manifestFields, segments, outPath) {
   const formatVersion = FORMAT_VERSIONS[manifestFields.profile];
   if (!formatVersion) {
     throw new Error(`manifest.profile must be ${PROFILE_V1} or ${PROFILE_V2}, got ${manifestFields.profile}`);

package/complete/fusion.mjs

@@ -0,0 +1,72 @@
+// Vector/lexical rank fusion for hybrid retrieval — the one definition the
+// reader (index.mjs), the calibrator (pikelet/src/calibrate.mjs, which fits
+// coverage on the passage fusion ranks first) and the BEIR ladder
+// (benchmarks/beir/query-E.mjs) all call, so a fit-time or benchmark
+// ranking is the ranking the reader serves.
+//
+// Reciprocal-rank fusion over the exact-reranked vector order and the BM25
+// order: score = 1/(K + vRank) + lexicalWeight/(K + lexRank), lexical term
+// present only for records BM25 returned. Every lexical candidate joins the
+// vector pool (the reader reranks them exactly), so it always has a vRank.
+//
+// The vector-margin guard: when the vector top-1 leads the top-2 by at
+// least guardMargin of its own distance, it keeps rank 1 regardless of
+// fusion. Without it, with K=60 any record that appears in both lists
+// outranks a vector-only rank 1 (1/(60+v) + w/(60+l) > 1/60 for any finite
+// v when w=1), so a query whose terms match many chunks — a character name
+// in a novel — fills the top ranks with term matches and buries the
+// passage the encoder placed first. Measured on three sets with
+// scripts/fusion-sim.mjs before any default changed.
+
+export const FUSION_DEFAULTS = Object.freeze({
+    rrfK: 60,
+    lexicalWeight: 0.5,
+    guardMargin: 0.05,
+});
+
+/**
+ * @param {Array<{id: number, distance: number}>} vectorOrder hits sorted by
+ *   exact distance ascending (the rerank output), including every lexical
+ *   candidate.
+ * @param {number[]} lexicalIds record ids in BM25 order (after the cutoff).
+ * @param {{rrfK?: number, lexicalWeight?: number, guardMargin?: number}} [options]
+ * @returns the same hit objects in fused order.
+ */
+export function fuseCandidates(vectorOrder, lexicalIds, options = {}) {
+    const rrfK = options.rrfK ?? FUSION_DEFAULTS.rrfK;
+    const lexicalWeight = options.lexicalWeight ?? FUSION_DEFAULTS.lexicalWeight;
+    const guardMargin = options.guardMargin ?? FUSION_DEFAULTS.guardMargin;
+    if (!(rrfK > 0) || !(lexicalWeight >= 0) || !(guardMargin >= 0)) {
+        throw new Error('fuseCandidates: rrfK must be > 0, lexicalWeight and guardMargin >= 0');
+    }
+    if (!lexicalIds.length) return vectorOrder.slice();
+    const lexRank = new Map();
+    lexicalIds.forEach((id, i) => { if (!lexRank.has(id)) lexRank.set(id, i); });
+    let fused = vectorOrder
+        .map((hit, vRank) => ({
+            hit,
+            score: 1 / (rrfK + vRank) + (lexRank.has(hit.id) ? lexicalWeight / (rrfK + lexRank.get(hit.id)) : 0),
+        }))
+        .sort((a, b) => (b.score - a.score) || (a.hit.distance - b.hit.distance))
+        .map((entry) => entry.hit);
+    if (guardMargin > 0 && vectorOrder.length > 1) {
+        const [first, second] = vectorOrder;
+        // The margin is relative to the top-1's own distance, which needs a
+        // positive denominator. first.distance === 0 is not a degenerate case
+        // to skip but the STRONGEST possible top-1 -- cosine distance is
+        // 1 - dot, so a query matching passage text verbatim scores exactly 0
+        // -- and gating on `first.distance > 0` disabled the guard for exactly
+        // the hit it exists to protect (a worse top-1 at 0.001 was guarded, a
+        // perfect one at 0 was not). Any second distance strictly greater than
+        // a zero first is an infinite relative lead, so the guard fires.
+        // Distances are non-negative here (cosine/L2 as returned by the
+        // rerank); a metric admitting negatives would need an absolute margin.
+        const lead = first.distance > 0
+            ? (second.distance - first.distance) / first.distance
+            : (second.distance > 0 ? Infinity : 0);
+        if (lead >= guardMargin && fused[0] !== first) {
+            fused = [first, ...fused.filter((hit) => hit !== first)];
+        }
+    }
+    return fused;
+}

package/complete/index.d.ts

@@ -1,6 +1,6 @@
 /**
- * pikelet-wasm/complete — reader for complete .pancake artifacts
- * (spec/COMPLETE_PROFILE.md, profile pancake-complete-v1). Opens any kind
+ * pikelet-wasm/complete — reader for complete .pikelet artifacts
+ * (spec/COMPLETE_PROFILE.md, profile pikelet-complete-v1). Opens any kind
  * (1 student-inline, 2 declared-external, 3 inline-transformer) from a file
  * path in Node or a range source anywhere.
  */
@@ -74,7 +74,7 @@ export interface CompleteSearch {
     identity: string;
     /** Container header formatVersion (1 or 2). */
     formatVersion: number;
-    /** Manifest profile: 'pancake-complete-v1' | 'pancake-complete-v2'. */
+    /** Manifest profile: 'pikelet-complete-v1' | 'pikelet-complete-v2'. */
     profile: string;
     /**
      * Human-readable pack name from the identity-verified manifest
@@ -158,6 +158,13 @@ export interface CompleteSearch {
     parallelism?: number;
     gap?: number;
     maxRangeBytes?: number;
+    /**
+     * A matchQuality: 'none' verdict withholds results by default. Set
+     * true to see the raw retrieval anyway (e.g. to inspect why the
+     * classifier abstained) — matchQuality and confidence are unaffected,
+     * this only controls whether results is populated. Default false.
+     */
+    showAbstained?: boolean;
   }): Promise<CompleteQueryResult>;
   /** Hydrate one corpus record by id (verified per record on format 2). */
   record(id: number): Promise<Record<string, unknown>>;
@@ -166,7 +173,7 @@ export interface CompleteSearch {
   close(): Promise<void>;
 }
 
-export declare function openPancakeFile(
+export declare function openPikeletFile(
   input: string | CompleteRangeSource,
   options?: {
     /**
@@ -238,6 +245,29 @@ export declare function openPancakeFile(
 
 /** Header formatVersion -> manifest profile accepted by this reader. */
 export declare const SUPPORTED_PROFILES: Readonly<Record<number, string>>;
+
+/** Rank-fusion parameters shared by the reader, the calibrator and the BEIR ladder. */
+export interface FusionOptions {
+  /** Reciprocal-rank-fusion constant (default 60). */
+  rrfK?: number;
+  /** Weight of the lexical (BM25) rank term relative to the vector term (default 0.5). */
+  lexicalWeight?: number;
+  /**
+   * Relative distance margin (top-2 minus top-1, over top-1) at or above which
+   * the vector top-1 keeps rank 1 (default 0.05; 0 disables the guard).
+   */
+  guardMargin?: number;
+}
+export declare const FUSION_DEFAULTS: Readonly<Required<FusionOptions>>;
+/**
+ * Fuse the exact-reranked vector order (every lexical candidate included) with the BM25 order.
+ * Returns the same hit objects, reordered.
+ */
+export declare function fuseCandidates<T extends { id: number; distance: number }>(
+  vectorOrder: T[],
+  lexicalIds: number[],
+  options?: FusionOptions,
+): T[];
 export declare const CORPUS_LAYOUT_V2: 'records-v2';
 /**
  * Check a host encoder against a kind-2 declaration's verification vectors
@@ -284,6 +314,3 @@ export declare function buildInlineTestVectors(embedder: unknown, texts?: string
   text: string; windows: number; embedding: number[]; tolerance: number;
 }>>;
 export declare function verifyInlineTestVectors(embedder: unknown): Promise<{ checked: number }>;
-
-/** Documented alias for openPancakeFile — the project renamed to Pikelet; the reader's original export stays for compatibility. */
-export declare const openPikeletFile: typeof openPancakeFile;