pi-mega-compact 0.4.21 → 0.4.24

package/dist/src/minilm.js

@@ -1,92 +0,0 @@
-/**
- * minilm.ts — local MiniLM (all-MiniLM-L6-v2) sentence embedder (Sprint 12).
- *
- * Implements the `Embedder` interface so it drops into the existing VectorStore
- * dedup cascade and search with no call-site changes. Inference is 100% local:
- * the ONNX model + WordPiece vocab are on-disk artifacts fetched once by
- * scripts/setup-minilm.mjs. There is NO network call at runtime (PREVENT-PI-004).
- *
- * Inputs (dynamic): input_ids, attention_mask, token_type_ids (int64).
- * Output: last_hidden_state (batch, seq, 384). We mean-pool over non-padded
- * tokens (attention_mask == 1) and L2-normalize → 384-dim unit vector.
- *
- * The ONNX session + tokenizer are loaded LAZILY on first embed() so the default
- * TrigramEmbedder path (and its zero native-init cost) is untouched unless
- * MEGACOMPACT_EMBEDDER=minilm is selected.
- */
-import { join } from "node:path";
-import { homedir } from "node:os";
-import { existsSync } from "node:fs";
-import { l2Normalize, awaitSync } from "./embedder.js";
-import { WordPieceTokenizer } from "./wordpiece.js";
-export const MINILM_DIM = 384;
-export const MINILM_MAX_LEN = 256;
-/** Resolve the model directory: MEGACOMPACT_MINILM_DIR > ./models/minilm > ~/.pi … */
-function resolveModelDir() {
-    if (process.env.MEGACOMPACT_MINILM_DIR)
-        return process.env.MEGACOMPACT_MINILM_DIR;
-    // Repo-local vendored path (gitignored).
-    const local = join(process.cwd(), "models", "minilm");
-    if (existsSync(local))
-        return local;
-    return join(homedir(), ".pi", "agent", "extensions", "mega-compact", "models", "minilm");
-}
-export class MiniLMEmbedder {
-    dim = MINILM_DIM;
-    session = null;
-    tokenizer = null;
-    modelDir;
-    loadPromise = null;
-    constructor(modelDir = resolveModelDir()) {
-        this.modelDir = modelDir;
-    }
-    async ensureLoaded() {
-        if (this.session && this.tokenizer)
-            return;
-        if (this.loadPromise)
-            return this.loadPromise;
-        this.loadPromise = (async () => {
-            const ort = await import("onnxruntime-node");
-            const modelPath = join(this.modelDir, "model_quantized.onnx");
-            const vocabPath = join(this.modelDir, "vocab.txt");
-            if (!existsSync(modelPath) || !existsSync(vocabPath)) {
-                throw new Error(`MiniLM artifacts missing in ${this.modelDir}. Run: node scripts/setup-minilm.mjs`);
-            }
-            // 1 thread is plenty for a single short-region embed and bounds CPU.
-            this.session = await ort.InferenceSession.create(modelPath, {
-                executionProviders: ["cpu"],
-                graphOptimizationLevel: "all",
-            });
-            this.tokenizer = WordPieceTokenizer.fromVocabFile(vocabPath);
-        })();
-        return this.loadPromise;
-    }
-    embed(text) {
-        awaitSync(this.ensureLoaded());
-        const enc = this.tokenizer.encode(text, MINILM_MAX_LEN);
-        const n = enc.inputIds.length;
-        const BigInt64 = (arr) => arr.map((x) => BigInt(x));
-        const ort = awaitSync(import("onnxruntime-node"));
-        const tensors = {
-            input_ids: new ort.Tensor("int64", BigInt64(enc.inputIds), [1, n]),
-            attention_mask: new ort.Tensor("int64", BigInt64(enc.attentionMask), [1, n]),
-            token_type_ids: new ort.Tensor("int64", BigInt64(enc.tokenTypeIds), [1, n]),
-        };
-        const out = awaitSync(this.session.run(tensors));
-        const hidden = out.last_hidden_state.data;
-        // hidden shape: [1, n, 384]. Mean-pool over non-padded positions.
-        const pooled = new Array(MINILM_DIM).fill(0);
-        let count = 0;
-        for (let i = 0; i < n; i++) {
-            if (enc.attentionMask[i] === 0)
-                continue;
-            const base = i * MINILM_DIM;
-            for (let d = 0; d < MINILM_DIM; d++)
-                pooled[d] += hidden[base + d];
-            count++;
-        }
-        if (count === 0)
-            return l2Normalize(new Array(MINILM_DIM).fill(0));
-        return l2Normalize(pooled.map((x) => x / count));
-    }
-}

package/dist/src/wordpiece.js

@@ -1,129 +0,0 @@
-/**
- * wordpiece.ts — a self-contained WordPiece tokenizer for BERT/MiniLM.
- *
- * Loads the canonical `vocab.txt` (bert-base-uncased, ~30K tokens) from disk and
- * implements the standard uncased BERT preprocessing + greedy longest-match
- * WordPiece segmentation. No native dependency, no network — the vocab file is a
- * local artifact fetched once by scripts/setup-minilm.mjs (PREVENT-PI-004).
- *
- * This mirrors HuggingFace `BertTokenizer` closely enough for sentence-embedding
- * use: lowercase, strip accents, split on whitespace + punctuation, then
- * WordPiece each token with the `##` continuation convention. Special tokens
- * [CLS]/[SEP] are added by the caller's encode().
- */
-import { readFileSync, existsSync } from "node:fs";
-const UNK = "[UNK]";
-const CLS = "[CLS]";
-const SEP = "[SEP]";
-const PAD = "[PAD]";
-const MAX_INPUT_CHARS_PER_WORD = 200;
-export class WordPieceTokenizer {
-    vocab;
-    clsId;
-    sepId;
-    padId;
-    unkId;
-    constructor(vocab) {
-        this.vocab = vocab;
-        this.clsId = vocab.get(CLS) ?? 101;
-        this.sepId = vocab.get(SEP) ?? 102;
-        this.padId = vocab.get(PAD) ?? 0;
-        this.unkId = vocab.get(UNK) ?? 100;
-    }
-    /** Build a tokenizer from a vocab.txt file (one token per line, index = line). */
-    static fromVocabFile(path) {
-        if (!existsSync(path)) {
-            throw new Error(`WordPiece vocab not found at ${path}. Run: node scripts/setup-minilm.mjs`);
-        }
-        const lines = readFileSync(path, "utf-8").split("\n");
-        const vocab = new Map();
-        for (let i = 0; i < lines.length; i++) {
-            const tok = lines[i].replace(/\r$/, "");
-            if (tok.length > 0 || i < lines.length - 1)
-                vocab.set(tok, i);
-        }
-        return new WordPieceTokenizer(vocab);
-    }
-    /** Uncased BERT basic tokenization: lowercase, strip accents, split on ws+punct. */
-    basicTokenize(text) {
-        // NFD + strip combining marks (accent removal), then lowercase.
-        const cleaned = text
-            .normalize("NFD")
-            .replace(/[̀-ͯ]/g, "")
-            .toLowerCase();
-        const tokens = [];
-        let buf = "";
-        const flush = () => {
-            if (buf.length > 0) {
-                tokens.push(buf);
-                buf = "";
-            }
-        };
-        for (const ch of cleaned) {
-            if (/\s/.test(ch)) {
-                flush();
-            }
-            else if (/[!-/:-@[-`{-~¡-¿]/.test(ch)) {
-                // Punctuation becomes its own token.
-                flush();
-                tokens.push(ch);
-            }
-            else {
-                buf += ch;
-            }
-        }
-        flush();
-        return tokens;
-    }
-    /** Greedy longest-match WordPiece for a single word. */
-    wordpiece(word) {
-        if (word.length > MAX_INPUT_CHARS_PER_WORD)
-            return [UNK];
-        const pieces = [];
-        let start = 0;
-        while (start < word.length) {
-            let end = word.length;
-            let cur = null;
-            while (start < end) {
-                let sub = word.slice(start, end);
-                if (start > 0)
-                    sub = "##" + sub;
-                if (this.vocab.has(sub)) {
-                    cur = sub;
-                    break;
-                }
-                end--;
-            }
-            if (cur === null)
-                return [UNK]; // any unmatchable piece → whole word is UNK
-            pieces.push(cur);
-            start = end;
-        }
-        return pieces;
-    }
-    /** Tokenize text into WordPiece token strings (no special tokens). */
-    tokenize(text) {
-        const out = [];
-        for (const word of this.basicTokenize(text)) {
-            for (const piece of this.wordpiece(word))
-                out.push(piece);
-        }
-        return out;
-    }
-    /**
-     * Encode text into model inputs with [CLS]…[SEP], truncated to `maxLen`.
-     * attention_mask is all 1s (no padding for single-sequence inference).
-     */
-    encode(text, maxLen = 256) {
-        const pieces = this.tokenize(text).slice(0, Math.max(0, maxLen - 2));
-        const inputIds = [this.clsId];
-        for (const p of pieces)
-            inputIds.push(this.vocab.get(p) ?? this.unkId);
-        inputIds.push(this.sepId);
-        return {
-            inputIds,
-            attentionMask: inputIds.map(() => 1),
-            tokenTypeIds: inputIds.map(() => 0),
-        };
-    }
-}