rehydra 0.1.0

package/dist/ner/tokenizer.js

@@ -0,0 +1,332 @@
+/**
+ * HuggingFace Tokenizer
+ * Loads and uses tokenizers from HuggingFace's tokenizer.json format
+ * Supports Unigram (SentencePiece) and BPE tokenizers
+ */
+/**
+ * Default tokenizer configuration
+ */
+export const DEFAULT_TOKENIZER_CONFIG = {
+    maxLength: 512,
+    doLowerCase: false, // XLM-RoBERTa doesn't lowercase
+};
+/**
+ * WordPiece Tokenizer - supports both HuggingFace JSON and vocab.txt formats
+ */
+export class WordPieceTokenizer {
+    vocab;
+    inverseVocab;
+    config;
+    sortedVocab;
+    // Special token IDs (XLM-RoBERTa style)
+    clsId = 0; // <s>
+    sepId = 2; // </s>
+    padId = 1; // <pad>
+    unkId = 3; // <unk>
+    // Special token strings
+    clsToken = '<s>';
+    sepToken = '</s>';
+    padToken = '<pad>';
+    unkToken = '<unk>';
+    constructor(vocab, config = {}) {
+        this.vocab = vocab;
+        this.config = { ...DEFAULT_TOKENIZER_CONFIG, ...config };
+        // Build inverse vocab
+        this.inverseVocab = new Map();
+        for (const [token, id] of vocab) {
+            this.inverseVocab.set(id, token);
+        }
+        // Sort vocab by token length (longest first) for greedy matching
+        this.sortedVocab = Array.from(vocab.entries()).sort((a, b) => b[0].length - a[0].length);
+        // Try to detect special tokens from vocab
+        this.detectSpecialTokens();
+    }
+    /**
+     * Detect special tokens from vocabulary
+     */
+    detectSpecialTokens() {
+        // XLM-RoBERTa style
+        if (this.vocab.has('<s>')) {
+            this.clsToken = '<s>';
+            this.clsId = this.vocab.get('<s>') ?? 0;
+            this.sepToken = '</s>';
+            this.sepId = this.vocab.get('</s>') ?? 2;
+            this.padToken = '<pad>';
+            this.padId = this.vocab.get('<pad>') ?? 1;
+            this.unkToken = '<unk>';
+            this.unkId = this.vocab.get('<unk>') ?? 3;
+        }
+        // BERT style
+        else if (this.vocab.has('[CLS]')) {
+            this.clsToken = '[CLS]';
+            this.clsId = this.vocab.get('[CLS]') ?? 101;
+            this.sepToken = '[SEP]';
+            this.sepId = this.vocab.get('[SEP]') ?? 102;
+            this.padToken = '[PAD]';
+            this.padId = this.vocab.get('[PAD]') ?? 0;
+            this.unkToken = '[UNK]';
+            this.unkId = this.vocab.get('[UNK]') ?? 100;
+        }
+    }
+    /**
+     * Tokenizes text into tokens with offset tracking
+     */
+    tokenize(text) {
+        const tokens = [];
+        const tokenToCharSpan = [];
+        // Add CLS token
+        tokens.push({
+            id: this.clsId,
+            token: this.clsToken,
+            start: 0,
+            end: 0,
+            isContinuation: false,
+            isSpecial: true,
+        });
+        tokenToCharSpan.push(null);
+        // Preprocess text
+        const processedText = this.config.doLowerCase ? text.toLowerCase() : text;
+        // Tokenize using greedy longest-match
+        let pos = 0;
+        while (pos < processedText.length) {
+            // Skip whitespace
+            if (/\s/.test(processedText[pos])) {
+                pos++;
+                continue;
+            }
+            // Find the longest matching token starting at this position
+            const { token, id, length } = this.findBestToken(processedText, pos);
+            const isFirstOfWord = pos === 0 || /\s/.test(processedText[pos - 1]);
+            tokens.push({
+                id,
+                token,
+                start: pos,
+                end: pos + length,
+                isContinuation: !isFirstOfWord && !token.startsWith('▁'),
+                isSpecial: false,
+            });
+            tokenToCharSpan.push([pos, pos + length]);
+            pos += length;
+        }
+        // Add SEP token
+        tokens.push({
+            id: this.sepId,
+            token: this.sepToken,
+            start: text.length,
+            end: text.length,
+            isContinuation: false,
+            isSpecial: true,
+        });
+        tokenToCharSpan.push(null);
+        // Truncate if necessary
+        const maxTokens = this.config.maxLength;
+        if (tokens.length > maxTokens) {
+            tokens.length = maxTokens - 1;
+            tokenToCharSpan.length = maxTokens - 1;
+            tokens.push({
+                id: this.sepId,
+                token: this.sepToken,
+                start: text.length,
+                end: text.length,
+                isContinuation: false,
+                isSpecial: true,
+            });
+            tokenToCharSpan.push(null);
+        }
+        // Build arrays
+        const inputIds = tokens.map((t) => t.id);
+        const attentionMask = tokens.map(() => 1);
+        const tokenTypeIds = tokens.map(() => 0);
+        return {
+            tokens,
+            inputIds,
+            attentionMask,
+            tokenTypeIds,
+            tokenToCharSpan,
+        };
+    }
+    /**
+     * Find the best matching token using greedy longest-match
+     */
+    findBestToken(text, startPos) {
+        const remaining = text.slice(startPos);
+        // Check if this starts a new word (preceded by space or start)
+        const isWordStart = startPos === 0 || /\s/.test(text[startPos - 1]);
+        // For SentencePiece models, word-initial tokens start with ▁
+        if (isWordStart) {
+            // Try with ▁ prefix first
+            const withPrefix = '▁' + remaining;
+            for (const [vocabToken, id] of this.sortedVocab) {
+                if (withPrefix.startsWith(vocabToken)) {
+                    // Return the match length without the ▁ since that's not in original text
+                    return {
+                        token: vocabToken,
+                        id,
+                        length: vocabToken.length - 1 // Subtract 1 for the ▁
+                    };
+                }
+            }
+        }
+        // Try exact match without prefix
+        for (const [vocabToken, id] of this.sortedVocab) {
+            // Skip special tokens and tokens starting with ▁ for non-word-start positions
+            if (vocabToken.startsWith('<') || vocabToken.startsWith('['))
+                continue;
+            if (!isWordStart && vocabToken.startsWith('▁'))
+                continue;
+            if (remaining.startsWith(vocabToken.replace(/^▁/, ''))) {
+                const matchLength = vocabToken.replace(/^▁/, '').length;
+                if (matchLength > 0) {
+                    return { token: vocabToken, id, length: matchLength };
+                }
+            }
+        }
+        // Single character fallback
+        const char = remaining[0];
+        const charId = this.vocab.get(char) ?? this.vocab.get('▁' + char) ?? this.unkId;
+        return { token: char, id: charId, length: 1 };
+    }
+    /**
+     * Decodes token IDs back to text
+     */
+    decode(tokenIds) {
+        const parts = [];
+        for (const id of tokenIds) {
+            const token = this.inverseVocab.get(id);
+            if (token === undefined)
+                continue;
+            if (token === this.clsToken || token === this.sepToken || token === this.padToken)
+                continue;
+            // SentencePiece uses ▁ to mark word boundaries
+            if (token.startsWith('▁')) {
+                parts.push(' ' + token.slice(1));
+            }
+            else {
+                parts.push(token);
+            }
+        }
+        return parts.join('').trim();
+    }
+    /**
+     * Gets vocabulary size
+     */
+    get vocabSize() {
+        return this.vocab.size;
+    }
+    /**
+     * Gets a token ID by string
+     */
+    getTokenId(token) {
+        return this.vocab.get(token);
+    }
+    /**
+     * Gets a token string by ID
+     */
+    getToken(id) {
+        return this.inverseVocab.get(id);
+    }
+}
+/**
+ * Loads vocabulary from a file (supports tokenizer.json and vocab.txt)
+ * Uses storage abstraction for browser compatibility
+ */
+export async function loadVocabFromFile(filePath) {
+    const { getStorageProvider } = await import('../utils/storage.js');
+    const storage = await getStorageProvider();
+    const content = await storage.readTextFile(filePath);
+    // Detect format
+    if (filePath.endsWith('.json') || content.trim().startsWith('{')) {
+        return parseHFTokenizerJson(content);
+    }
+    else {
+        return parseVocab(content);
+    }
+}
+/**
+ * Loads vocabulary from content string (for when content is already available)
+ */
+export function loadVocabFromContent(content, format = 'json') {
+    if (format === 'json' || content.trim().startsWith('{')) {
+        return parseHFTokenizerJson(content);
+    }
+    else {
+        return parseVocab(content);
+    }
+}
+/**
+ * Parses HuggingFace tokenizer.json format
+ */
+export function parseHFTokenizerJson(content) {
+    const vocab = new Map();
+    try {
+        const config = JSON.parse(content);
+        // Add special tokens first
+        if (Array.isArray(config.added_tokens)) {
+            for (const token of config.added_tokens) {
+                vocab.set(token.content, token.id);
+            }
+        }
+        // Add model vocabulary
+        if (config.model !== undefined && config.model.vocab !== undefined) {
+            if (Array.isArray(config.model.vocab)) {
+                // Unigram format: array of [token, score] pairs
+                for (let i = 0; i < config.model.vocab.length; i++) {
+                    const entry = config.model.vocab[i];
+                    if (entry && typeof entry[0] === 'string') {
+                        vocab.set(entry[0], i);
+                    }
+                }
+            }
+            else {
+                // BPE/WordPiece format: object mapping token -> id
+                for (const [token, id] of Object.entries(config.model.vocab)) {
+                    vocab.set(token, id);
+                }
+            }
+        }
+    }
+    catch (e) {
+        throw new Error(`Failed to parse tokenizer.json: ${String(e)}`);
+    }
+    return vocab;
+}
+/**
+ * Parses vocabulary from string content (vocab.txt format)
+ */
+export function parseVocab(content) {
+    const vocab = new Map();
+    const lines = content.split('\n');
+    for (let i = 0; i < lines.length; i++) {
+        const token = lines[i]?.trim();
+        if (token !== undefined && token.length > 0) {
+            vocab.set(token, i);
+        }
+    }
+    return vocab;
+}
+/**
+ * Creates a minimal vocabulary for testing
+ */
+export function createTestVocab() {
+    const tokens = [
+        '<s>',
+        '<pad>',
+        '</s>',
+        '<unk>',
+        '▁Hello',
+        '▁John',
+        '▁Smith',
+        '▁from',
+        '▁Acme',
+        '▁Corp',
+        '▁in',
+        '▁Berlin',
+        '!',
+    ];
+    const vocab = new Map();
+    tokens.forEach((token, index) => {
+        vocab.set(token, index);
+    });
+    return vocab;
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/ner/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/ner/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAkEH;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACvD,SAAS,EAAE,GAAG;IACd,WAAW,EAAE,KAAK,EAAE,gCAAgC;CACrD,CAAC;AAEF;;GAEG;AACH,MAAM,OAAO,kBAAkB;IACrB,KAAK,CAAsB;IAC3B,YAAY,CAAsB;IAClC,MAAM,CAAkB;IACxB,WAAW,CAA0B;IAE7C,wCAAwC;IAChC,KAAK,GAAW,CAAC,CAAC,CAAE,MAAM;IAC1B,KAAK,GAAW,CAAC,CAAC,CAAE,OAAO;IAC3B,KAAK,GAAW,CAAC,CAAC,CAAE,QAAQ;IAC5B,KAAK,GAAW,CAAC,CAAC,CAAE,QAAQ;IAEpC,wBAAwB;IAChB,QAAQ,GAAW,KAAK,CAAC;IACzB,QAAQ,GAAW,MAAM,CAAC;IAC1B,QAAQ,GAAW,OAAO,CAAC;IAC3B,QAAQ,GAAW,OAAO,CAAC;IAEnC,YAAY,KAA0B,EAAE,SAAmC,EAAE;QAC3E,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAC;QAEzD,sBAAsB;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QAED,iEAAiE;QACjE,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAEzF,0CAA0C;QAC1C,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACK,mBAAmB;QACzB,oBAAoB;QACpB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;YACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC;YACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;YACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;YACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;QACD,aAAa;aACR,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;YACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;YAC5C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;YACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;YAC5C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;YACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;YACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;QAC9C,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,IAAY;QACnB,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,eAAe,GAAmC,EAAE,CAAC;QAE3D,gBAAgB;QAChB,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,IAAI,CAAC,KAAK;YACd,KAAK,EAAE,IAAI,CAAC,QAAQ;YACpB,KAAK,EAAE,CAAC;YACR,GAAG,EAAE,CAAC;YACN,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE3B,kBAAkB;QAClB,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAE1E,sCAAsC;QACtC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,OAAO,GAAG,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC;YAClC,kBAAkB;YAClB,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAE,CAAC,EAAE,CAAC;gBACnC,GAAG,EAAE,CAAC;gBACN,SAAS;YACX,CAAC;YAED,4DAA4D;YAC5D,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YAErE,MAAM,aAAa,GAAG,GAAG,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC,CAAC;YAEtE,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE;gBACF,KAAK;gBACL,KAAK,EAAE,GAAG;gBACV,GAAG,EAAE,GAAG,GAAG,MAAM;gBACjB,cAAc,EAAE,CAAC,aAAa,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;gBACxD,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC;YAE1C,GAAG,IAAI,MAAM,CAAC;QAChB,CAAC;QAED,gBAAgB;QAChB,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,IAAI,CAAC,KAAK;YACd,KAAK,EAAE,IAAI,CAAC,QAAQ;YACpB,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,GAAG,EAAE,IAAI,CAAC,MAAM;YAChB,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE3B,wBAAwB;QACxB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;QACxC,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC;YAC9B,eAAe,CAAC,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,IAAI,CAAC,KAAK;gBACd,KAAK,EAAE,IAAI,CAAC,QAAQ;gBACpB,KAAK,EAAE,IAAI,CAAC,MAAM;gBAClB,GAAG,EAAE,IAAI,CAAC,MAAM;gBAChB,cAAc,EAAE,KAAK;gBACrB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YACH,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAED,eAAe;QACf,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAEzC,OAAO;YACL,MAAM;YACN,QAAQ;YACR,aAAa;YACb,YAAY;YACZ,eAAe;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,IAAY,EAAE,QAAgB;QAClD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEvC,+DAA+D;QAC/D,MAAM,WAAW,GAAG,QAAQ,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAE,CAAC,CAAC;QAErE,6DAA6D;QAC7D,IAAI,WAAW,EAAE,CAAC;YAChB,0BAA0B;YAC1B,MAAM,UAAU,GAAG,GAAG,GAAG,SAAS,CAAC;YACnC,KAAK,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAChD,IAAI,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;oBACtC,0EAA0E;oBAC1E,OAAO;wBACL,KAAK,EAAE,UAAU;wBACjB,EAAE;wBACF,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,uBAAuB;qBACtD,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,KAAK,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAChD,8EAA8E;YAC9E,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YACvE,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEzD,IAAI,SAAS,CAAC,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;gBACvD,MAAM,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC;gBACxD,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;oBACpB,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC;QAChF,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,QAAkB;QACvB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxC,IAAI,KAAK,KAAK,SAAS;gBAAE,SAAS;YAClC,IAAI,KAAK,KAAK,IAAI,CAAC,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAE5F,+CAA+C;YAC/C,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,EAAU;QACjB,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,MAAM,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IAErD,gBAAgB;IAChB,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACjE,OAAO,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe,EAAE,SAAyB,MAAM;IACnF,IAAI,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACxD,OAAO,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAExC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAsB,CAAC;QAExD,2BAA2B;QAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;YACvC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBACxC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACnE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtC,gDAAgD;gBAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACnD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACpC,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;wBAC1C,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBACzB,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,mDAAmD;gBACnD,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,mCAAmC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,OAAe;IACxC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QAC/B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG;QACb,KAAK;QACL,OAAO;QACP,MAAM;QACN,OAAO;QACP,QAAQ;QACR,OAAO;QACP,QAAQ;QACR,OAAO;QACP,OAAO;QACP,OAAO;QACP,KAAK;QACL,SAAS;QACT,GAAG;KACJ,CAAC;IAEF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/pipeline/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Pipeline Module
+ * Exports all pipeline components
+ */
+export * from "./prenormalize.js";
+export * from "./resolver.js";
+export * from "./tagger.js";
+export * from "./validator.js";
+export * from "./semantic-enricher.js";
+export * from "./semantic-data-loader.js";
+export * from "./title-extractor.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/pipeline/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sBAAsB,CAAC"}

package/dist/pipeline/index.js

@@ -0,0 +1,12 @@
+/**
+ * Pipeline Module
+ * Exports all pipeline components
+ */
+export * from "./prenormalize.js";
+export * from "./resolver.js";
+export * from "./tagger.js";
+export * from "./validator.js";
+export * from "./semantic-enricher.js";
+export * from "./semantic-data-loader.js";
+export * from "./title-extractor.js";
+//# sourceMappingURL=index.js.map

package/dist/pipeline/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sBAAsB,CAAC"}

package/dist/pipeline/prenormalize.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Pre-normalization
+ * Normalizes text before PII detection
+ */
+/**
+ * Pre-normalization options
+ */
+export interface PrenormalizeOptions {
+    /** Normalize line endings to \n */
+    normalizeLineEndings: boolean;
+    /** Apply Unicode NFKC normalization */
+    unicodeNormalize: boolean;
+    /** Trim leading/trailing whitespace */
+    trim: boolean;
+}
+/**
+ * Default pre-normalization options
+ */
+export declare const DEFAULT_PRENORMALIZE_OPTIONS: PrenormalizeOptions;
+/**
+ * Pre-normalizes text for PII detection
+ * Note: This currently only normalizes line endings to preserve character offsets
+ *
+ * @param text - Original input text
+ * @param options - Normalization options
+ * @returns Normalized text
+ */
+export declare function prenormalize(text: string, options?: Partial<PrenormalizeOptions>): string;
+/**
+ * Calculates offset adjustments when text is modified
+ * Used when prenormalization changes text length
+ */
+export interface OffsetMapping {
+    /** Map from original offset to normalized offset */
+    toNormalized: (originalOffset: number) => number;
+    /** Map from normalized offset to original offset */
+    toOriginal: (normalizedOffset: number) => number;
+}
+/**
+ * Creates an identity offset mapping (no changes)
+ */
+export declare function createIdentityMapping(): OffsetMapping;
+/**
+ * Creates offset mapping for line ending normalization
+ * This handles \r\n -> \n replacement
+ */
+export declare function createLineEndingMapping(originalText: string): OffsetMapping;
+//# sourceMappingURL=prenormalize.d.ts.map

package/dist/pipeline/prenormalize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prenormalize.d.ts","sourceRoot":"","sources":["../../src/pipeline/prenormalize.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,mCAAmC;IACnC,oBAAoB,EAAE,OAAO,CAAC;IAC9B,uCAAuC;IACvC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,uCAAuC;IACvC,IAAI,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,eAAO,MAAM,4BAA4B,EAAE,mBAI1C,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,OAAO,CAAC,mBAAmB,CAAM,GACzC,MAAM,CAqBR;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,oDAAoD;IACpD,YAAY,EAAE,CAAC,cAAc,EAAE,MAAM,KAAK,MAAM,CAAC;IACjD,oDAAoD;IACpD,UAAU,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,MAAM,CAAC;CAClD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,aAAa,CAKrD;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,MAAM,GAAG,aAAa,CA4C3E"}

package/dist/pipeline/prenormalize.js

@@ -0,0 +1,94 @@
+/**
+ * Pre-normalization
+ * Normalizes text before PII detection
+ */
+/**
+ * Default pre-normalization options
+ */
+export const DEFAULT_PRENORMALIZE_OPTIONS = {
+    normalizeLineEndings: true,
+    unicodeNormalize: false, // Disabled by default to preserve offsets
+    trim: false, // Disabled by default to preserve offsets
+};
+/**
+ * Pre-normalizes text for PII detection
+ * Note: This currently only normalizes line endings to preserve character offsets
+ *
+ * @param text - Original input text
+ * @param options - Normalization options
+ * @returns Normalized text
+ */
+export function prenormalize(text, options = {}) {
+    const opts = { ...DEFAULT_PRENORMALIZE_OPTIONS, ...options };
+    let result = text;
+    // Normalize line endings (\r\n -> \n, \r -> \n)
+    if (opts.normalizeLineEndings) {
+        result = result.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+    }
+    // Note: Unicode normalization (NFKC) can change string length
+    // We skip it by default to preserve character offsets
+    // If needed, implement offset mapping
+    if (opts.unicodeNormalize) {
+        result = result.normalize('NFKC');
+    }
+    if (opts.trim) {
+        result = result.trim();
+    }
+    return result;
+}
+/**
+ * Creates an identity offset mapping (no changes)
+ */
+export function createIdentityMapping() {
+    return {
+        toNormalized: (offset) => offset,
+        toOriginal: (offset) => offset,
+    };
+}
+/**
+ * Creates offset mapping for line ending normalization
+ * This handles \r\n -> \n replacement
+ */
+export function createLineEndingMapping(originalText) {
+    // Find all \r\n positions
+    const crlfPositions = [];
+    for (let i = 0; i < originalText.length - 1; i++) {
+        if (originalText[i] === '\r' && originalText[i + 1] === '\n') {
+            crlfPositions.push(i);
+        }
+    }
+    if (crlfPositions.length === 0) {
+        return createIdentityMapping();
+    }
+    return {
+        toNormalized(originalOffset) {
+            // Count how many \r\n pairs are before this offset
+            let adjustment = 0;
+            for (const pos of crlfPositions) {
+                if (pos < originalOffset) {
+                    adjustment++;
+                }
+                else {
+                    break;
+                }
+            }
+            return originalOffset - adjustment;
+        },
+        toOriginal(normalizedOffset) {
+            // Add back the removed \r characters
+            let adjustment = 0;
+            let currentNormalized = 0;
+            for (const pos of crlfPositions) {
+                if (currentNormalized + (pos - adjustment) <= normalizedOffset) {
+                    adjustment++;
+                    currentNormalized = pos - adjustment + 1;
+                }
+                else {
+                    break;
+                }
+            }
+            return normalizedOffset + adjustment;
+        },
+    };
+}
+//# sourceMappingURL=prenormalize.js.map

package/dist/pipeline/prenormalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prenormalize.js","sourceRoot":"","sources":["../../src/pipeline/prenormalize.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAcH;;GAEG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAwB;IAC/D,oBAAoB,EAAE,IAAI;IAC1B,gBAAgB,EAAE,KAAK,EAAE,0CAA0C;IACnE,IAAI,EAAE,KAAK,EAAE,0CAA0C;CACxD,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,UAAwC,EAAE;IAE1C,MAAM,IAAI,GAAG,EAAE,GAAG,4BAA4B,EAAE,GAAG,OAAO,EAAE,CAAC;IAC7D,IAAI,MAAM,GAAG,IAAI,CAAC;IAElB,gDAAgD;IAChD,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC9B,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,8DAA8D;IAC9D,sDAAsD;IACtD,sCAAsC;IACtC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAaD;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACnC,OAAO;QACL,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM;QAChC,UAAU,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM;KAC/B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,YAAoB;IAC1D,0BAA0B;IAC1B,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,YAAY,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7D,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,qBAAqB,EAAE,CAAC;IACjC,CAAC;IAED,OAAO;QACL,YAAY,CAAC,cAAsB;YACjC,mDAAmD;YACnD,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBAChC,IAAI,GAAG,GAAG,cAAc,EAAE,CAAC;oBACzB,UAAU,EAAE,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YACD,OAAO,cAAc,GAAG,UAAU,CAAC;QACrC,CAAC;QAED,UAAU,CAAC,gBAAwB;YACjC,qCAAqC;YACrC,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBAChC,IAAI,iBAAiB,GAAG,CAAC,GAAG,GAAG,UAAU,CAAC,IAAI,gBAAgB,EAAE,CAAC;oBAC/D,UAAU,EAAE,CAAC;oBACb,iBAAiB,GAAG,GAAG,GAAG,UAAU,GAAG,CAAC,CAAC;gBAC3C,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,OAAO,gBAAgB,GAAG,UAAU,CAAC;QACvC,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/pipeline/resolver.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Entity Resolver
+ * Merges, deduplicates, and resolves overlapping entity detections
+ */
+import { SpanMatch, AnonymizationPolicy } from '../types/index.js';
+/**
+ * Resolution strategy for overlapping entities
+ */
+export declare enum OverlapStrategy {
+    /** Regex matches always win over NER */
+    REGEX_PRIORITY = "REGEX_PRIORITY",
+    /** Longer span wins */
+    LONGER_SPAN = "LONGER_SPAN",
+    /** Higher confidence wins */
+    HIGHER_CONFIDENCE = "HIGHER_CONFIDENCE",
+    /** Use type priority from policy */
+    TYPE_PRIORITY = "TYPE_PRIORITY"
+}
+/**
+ * Entity resolver configuration
+ */
+export interface ResolverConfig {
+    /** Primary strategy for overlap resolution */
+    overlapStrategy: OverlapStrategy;
+    /** Whether regex matches always take precedence */
+    regexPriority: boolean;
+    /** Minimum confidence to keep an entity */
+    minConfidence: number;
+}
+/**
+ * Default resolver configuration
+ */
+export declare const DEFAULT_RESOLVER_CONFIG: ResolverConfig;
+/**
+ * Resolves and merges entity detections from regex and NER
+ */
+export declare function resolveEntities(regexMatches: SpanMatch[], nerMatches: SpanMatch[], policy: AnonymizationPolicy, originalText: string, config?: Partial<ResolverConfig>): SpanMatch[];
+/**
+ * Creates protected spans from regex matches
+ * Used to mask regex matches from NER to avoid double-detection
+ */
+export declare function createProtectedSpans(regexMatches: SpanMatch[]): Array<{
+    start: number;
+    end: number;
+}>;
+/**
+ * Checks if a span overlaps with any protected span
+ */
+export declare function isInProtectedSpan(span: {
+    start: number;
+    end: number;
+}, protectedSpans: Array<{
+    start: number;
+    end: number;
+}>): boolean;
+//# sourceMappingURL=resolver.d.ts.map

package/dist/pipeline/resolver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/pipeline/resolver.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAEL,SAAS,EAET,mBAAmB,EAEpB,MAAM,mBAAmB,CAAC;AAG3B;;GAEG;AACH,oBAAY,eAAe;IACzB,wCAAwC;IACxC,cAAc,mBAAmB;IACjC,uBAAuB;IACvB,WAAW,gBAAgB;IAC3B,6BAA6B;IAC7B,iBAAiB,sBAAsB;IACvC,oCAAoC;IACpC,aAAa,kBAAkB;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8CAA8C;IAC9C,eAAe,EAAE,eAAe,CAAC;IACjC,mDAAmD;IACnD,aAAa,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,eAAO,MAAM,uBAAuB,EAAE,cAIrC,CAAC;AAEF;;GAEG;AACH,wBAAgB,eAAe,CAC7B,YAAY,EAAE,SAAS,EAAE,EACzB,UAAU,EAAE,SAAS,EAAE,EACvB,MAAM,EAAE,mBAAmB,EAC3B,YAAY,EAAE,MAAM,EACpB,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GACnC,SAAS,EAAE,CAyBb;AAkOD;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,YAAY,EAAE,SAAS,EAAE,GACxB,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAEvC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACpC,cAAc,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,GACpD,OAAO,CAET"}