pi-mega-compact 0.7.6 → 0.7.8

package/dist/src/wordpiece.js

@@ -1,129 +0,0 @@
-/**
- * wordpiece.ts — a self-contained WordPiece tokenizer for BERT/MiniLM.
- *
- * Loads the canonical `vocab.txt` (bert-base-uncased, ~30K tokens) from disk and
- * implements the standard uncased BERT preprocessing + greedy longest-match
- * WordPiece segmentation. No native dependency, no network — the vocab file is a
- * local artifact fetched once by scripts/setup-minilm.mjs (PREVENT-PI-004).
- *
- * This mirrors HuggingFace `BertTokenizer` closely enough for sentence-embedding
- * use: lowercase, strip accents, split on whitespace + punctuation, then
- * WordPiece each token with the `##` continuation convention. Special tokens
- * [CLS]/[SEP] are added by the caller's encode().
- */
-import { readFileSync, existsSync } from "node:fs";
-const UNK = "[UNK]";
-const CLS = "[CLS]";
-const SEP = "[SEP]";
-const PAD = "[PAD]";
-const MAX_INPUT_CHARS_PER_WORD = 200;
-export class WordPieceTokenizer {
-    vocab;
-    clsId;
-    sepId;
-    padId;
-    unkId;
-    constructor(vocab) {
-        this.vocab = vocab;
-        this.clsId = vocab.get(CLS) ?? 101;
-        this.sepId = vocab.get(SEP) ?? 102;
-        this.padId = vocab.get(PAD) ?? 0;
-        this.unkId = vocab.get(UNK) ?? 100;
-    }
-    /** Build a tokenizer from a vocab.txt file (one token per line, index = line). */
-    static fromVocabFile(path) {
-        if (!existsSync(path)) {
-            throw new Error(`WordPiece vocab not found at ${path}. Run: node scripts/setup-minilm.mjs`);
-        }
-        const lines = readFileSync(path, "utf-8").split("\n");
-        const vocab = new Map();
-        for (let i = 0; i < lines.length; i++) {
-            const tok = lines[i].replace(/\r$/, "");
-            if (tok.length > 0 || i < lines.length - 1)
-                vocab.set(tok, i);
-        }
-        return new WordPieceTokenizer(vocab);
-    }
-    /** Uncased BERT basic tokenization: lowercase, strip accents, split on ws+punct. */
-    basicTokenize(text) {
-        // NFD + strip combining marks (accent removal), then lowercase.
-        const cleaned = text
-            .normalize("NFD")
-            .replace(/[̀-ͯ]/g, "")
-            .toLowerCase();
-        const tokens = [];
-        let buf = "";
-        const flush = () => {
-            if (buf.length > 0) {
-                tokens.push(buf);
-                buf = "";
-            }
-        };
-        for (const ch of cleaned) {
-            if (/\s/.test(ch)) {
-                flush();
-            }
-            else if (/[!-/:-@[-`{-~¡-¿]/.test(ch)) {
-                // Punctuation becomes its own token.
-                flush();
-                tokens.push(ch);
-            }
-            else {
-                buf += ch;
-            }
-        }
-        flush();
-        return tokens;
-    }
-    /** Greedy longest-match WordPiece for a single word. */
-    wordpiece(word) {
-        if (word.length > MAX_INPUT_CHARS_PER_WORD)
-            return [UNK];
-        const pieces = [];
-        let start = 0;
-        while (start < word.length) {
-            let end = word.length;
-            let cur = null;
-            while (start < end) {
-                let sub = word.slice(start, end);
-                if (start > 0)
-                    sub = "##" + sub;
-                if (this.vocab.has(sub)) {
-                    cur = sub;
-                    break;
-                }
-                end--;
-            }
-            if (cur === null)
-                return [UNK]; // any unmatchable piece → whole word is UNK
-            pieces.push(cur);
-            start = end;
-        }
-        return pieces;
-    }
-    /** Tokenize text into WordPiece token strings (no special tokens). */
-    tokenize(text) {
-        const out = [];
-        for (const word of this.basicTokenize(text)) {
-            for (const piece of this.wordpiece(word))
-                out.push(piece);
-        }
-        return out;
-    }
-    /**
-     * Encode text into model inputs with [CLS]…[SEP], truncated to `maxLen`.
-     * attention_mask is all 1s (no padding for single-sequence inference).
-     */
-    encode(text, maxLen = 256) {
-        const pieces = this.tokenize(text).slice(0, Math.max(0, maxLen - 2));
-        const inputIds = [this.clsId];
-        for (const p of pieces)
-            inputIds.push(this.vocab.get(p) ?? this.unkId);
-        inputIds.push(this.sepId);
-        return {
-            inputIds,
-            attentionMask: inputIds.map(() => 1),
-            tokenTypeIds: inputIds.map(() => 0),
-        };
-    }
-}