rehydra 0.1.0

package/dist/ner/ner-model.js

@@ -0,0 +1,253 @@
+/**
+ * NER Model Wrapper
+ * ONNX Runtime integration for Named Entity Recognition
+ * Supports both onnxruntime-node and onnxruntime-web
+ */
+import { loadRuntime } from "./onnx-runtime.js";
+import { WordPieceTokenizer, loadVocabFromFile, } from "./tokenizer.js";
+import { decodeBIOTags, convertToSpanMatches, cleanupSpanBoundaries, mergeAdjacentSpans, } from "./bio-decoder.js";
+import { getStorageProvider, isBrowser } from "../utils/storage.js";
+/**
+ * Default label map for common NER models (CoNLL-style)
+ */
+export const DEFAULT_LABEL_MAP = [
+    "O",
+    "B-PER",
+    "I-PER",
+    "B-ORG",
+    "I-ORG",
+    "B-LOC",
+    "I-LOC",
+    "B-MISC",
+    "I-MISC",
+];
+/**
+ * NER Model wrapper for ONNX inference
+ */
+export class NERModel {
+    ort = null;
+    session = null;
+    tokenizer = null;
+    config;
+    isLoaded = false;
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Loads the model and tokenizer
+     */
+    async load() {
+        if (this.isLoaded)
+            return;
+        // Load ONNX runtime (auto-detects best runtime for environment)
+        this.ort = await loadRuntime();
+        // Load ONNX model
+        // In browsers, we need to load the model as ArrayBuffer since file paths don't work
+        // onnxruntime-web accepts ArrayBuffer/Uint8Array, while onnxruntime-node accepts file paths
+        if (isBrowser()) {
+            const storage = await getStorageProvider();
+            const modelData = await storage.readFile(this.config.modelPath);
+            // onnxruntime-web accepts Uint8Array directly
+            this.session = await this.ort.InferenceSession.create(modelData);
+        }
+        else {
+            // In Node.js, we can use the file path directly
+            this.session = await this.ort.InferenceSession.create(this.config.modelPath);
+        }
+        // Load tokenizer vocabulary (already uses storage abstraction internally)
+        const vocab = await loadVocabFromFile(this.config.vocabPath);
+        this.tokenizer = new WordPieceTokenizer(vocab, {
+            maxLength: this.config.maxLength,
+            doLowerCase: this.config.doLowerCase,
+        });
+        this.isLoaded = true;
+    }
+    /**
+     * Predicts entities in text
+     */
+    async predict(text, policy) {
+        const startTime = performance.now();
+        if (!this.isLoaded || this.session === null || this.tokenizer === null) {
+            throw new Error("Model not loaded. Call load() first.");
+        }
+        // Tokenize input
+        const tokenization = this.tokenizer.tokenize(text);
+        // Run inference
+        const { labels, confidences } = await this.runInference(tokenization);
+        // Decode BIO tags to entities
+        const rawEntities = decodeBIOTags(tokenization.tokens, labels, confidences, text);
+        // Convert to SpanMatch format with confidence filtering
+        const minConfidence = this.getMinConfidence(policy);
+        let spans = convertToSpanMatches(rawEntities, minConfidence);
+        // Post-process spans
+        spans = cleanupSpanBoundaries(spans, text);
+        spans = mergeAdjacentSpans(spans, text);
+        // Filter by enabled types in policy
+        if (policy !== undefined) {
+            spans = spans.filter((span) => policy.enabledTypes.has(span.type) &&
+                policy.nerEnabledTypes.has(span.type));
+        }
+        const endTime = performance.now();
+        return {
+            spans,
+            processingTimeMs: endTime - startTime,
+            modelVersion: this.config.modelVersion,
+        };
+    }
+    /**
+     * Runs ONNX inference
+     */
+    async runInference(tokenization) {
+        if (this.session === null || this.ort === null) {
+            throw new Error("Session not initialized");
+        }
+        const session = this.session;
+        const seqLength = tokenization.inputIds.length;
+        // Create tensors
+        const inputIdsTensor = new this.ort.Tensor("int64", BigInt64Array.from(tokenization.inputIds.map(BigInt)), [1, seqLength]);
+        const attentionMaskTensor = new this.ort.Tensor("int64", BigInt64Array.from(tokenization.attentionMask.map(BigInt)), [1, seqLength]);
+        const tokenTypeIdsTensor = new this.ort.Tensor("int64", BigInt64Array.from(tokenization.tokenTypeIds.map(BigInt)), [1, seqLength]);
+        // Run inference
+        const feeds = {
+            input_ids: inputIdsTensor,
+            attention_mask: attentionMaskTensor,
+        };
+        // Some models also need token_type_ids
+        const inputNames = session.inputNames;
+        if (inputNames.includes("token_type_ids")) {
+            feeds["token_type_ids"] = tokenTypeIdsTensor;
+        }
+        const results = await session.run(feeds);
+        // Get logits output
+        const outputName = session.outputNames[0];
+        if (outputName === undefined) {
+            throw new Error("No output from model");
+        }
+        const logits = results[outputName];
+        if (logits === undefined) {
+            throw new Error("Logits output not found");
+        }
+        // Process logits to get labels and confidences
+        return this.processLogits(logits, seqLength);
+    }
+    /**
+     * Processes model logits to extract labels and confidences
+     */
+    processLogits(logits, seqLength) {
+        const data = logits.data;
+        const numLabels = this.config.labelMap.length;
+        const labels = [];
+        const confidences = [];
+        for (let i = 0; i < seqLength; i++) {
+            // Get logits for this token
+            const tokenLogits = [];
+            for (let j = 0; j < numLabels; j++) {
+                tokenLogits.push(data[i * numLabels + j] ?? 0);
+            }
+            // Apply softmax
+            const probs = softmax(tokenLogits);
+            // Get argmax
+            let maxIdx = 0;
+            let maxProb = probs[0] ?? 0;
+            for (let j = 1; j < probs.length; j++) {
+                if ((probs[j] ?? 0) > maxProb) {
+                    maxProb = probs[j] ?? 0;
+                    maxIdx = j;
+                }
+            }
+            labels.push(this.config.labelMap[maxIdx] ?? "O");
+            confidences.push(maxProb);
+        }
+        return { labels, confidences };
+    }
+    /**
+     * Gets minimum confidence threshold from policy
+     */
+    getMinConfidence(policy) {
+        if (policy === undefined)
+            return 0.5;
+        // Get minimum from all NER-enabled types
+        let minThreshold = 1.0;
+        for (const type of policy.nerEnabledTypes) {
+            const threshold = policy.confidenceThresholds.get(type) ?? 0.5;
+            if (threshold < minThreshold) {
+                minThreshold = threshold;
+            }
+        }
+        return minThreshold;
+    }
+    /**
+     * Gets model version
+     */
+    get version() {
+        return this.config.modelVersion;
+    }
+    /**
+     * Checks if model is loaded
+     */
+    get loaded() {
+        return this.isLoaded;
+    }
+    /**
+     * Disposes of model resources
+     */
+    dispose() {
+        // ONNX Runtime Node doesn't have explicit dispose, but we can clear references
+        this.session = null;
+        this.tokenizer = null;
+        this.isLoaded = false;
+        return Promise.resolve();
+    }
+}
+/**
+ * Softmax function for probability calculation
+ */
+function softmax(logits) {
+    const maxLogit = Math.max(...logits);
+    const expLogits = logits.map((x) => Math.exp(x - maxLogit));
+    const sumExp = expLogits.reduce((a, b) => a + b, 0);
+    return expLogits.map((x) => x / sumExp);
+}
+/**
+ * Creates a NER model instance with configuration
+ */
+export function createNERModel(config) {
+    const fullConfig = {
+        modelPath: config.modelPath,
+        vocabPath: config.vocabPath,
+        labelMap: config.labelMap ?? DEFAULT_LABEL_MAP,
+        maxLength: config.maxLength ?? 512,
+        doLowerCase: config.doLowerCase ?? false, // XLM-RoBERTa is cased
+        modelVersion: config.modelVersion ?? "1.0.0",
+    };
+    return new NERModel(fullConfig);
+}
+/**
+ * NER Model stub for when no model is available
+ * Returns empty results - useful for regex-only mode
+ */
+export class NERModelStub {
+    version = "stub-1.0.0";
+    loaded = true;
+    async load() {
+        // No-op
+    }
+    predict(_text, _policy) {
+        return Promise.resolve({
+            spans: [],
+            processingTimeMs: 0,
+            modelVersion: this.version,
+        });
+    }
+    dispose() {
+        // No-op
+        return Promise.resolve();
+    }
+}
+/**
+ * Creates a stub NER model (for testing or regex-only mode)
+ */
+export function createNERModelStub() {
+    return new NERModelStub();
+}
+//# sourceMappingURL=ner-model.js.map

package/dist/ner/ner-model.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ner-model.js","sourceRoot":"","sources":["../../src/ner/ner-model.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,WAAW,EAAmB,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EACL,kBAAkB,EAClB,iBAAiB,GAElB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAgCpE;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,GAAG;IACH,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,QAAQ;IACR,QAAQ;CACT,CAAC;AAEF;;GAEG;AACH,MAAM,OAAO,QAAQ;IACX,GAAG,GAAsB,IAAI,CAAC;IAC9B,OAAO,GAAY,IAAI,CAAC;IACxB,SAAS,GAA8B,IAAI,CAAC;IAC5C,MAAM,CAAiB;IACvB,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAAY,MAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE1B,gEAAgE;QAChE,IAAI,CAAC,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;QAE/B,kBAAkB;QAClB,oFAAoF;QACpF,4FAA4F;QAC5F,IAAI,SAAS,EAAE,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,MAAM,kBAAkB,EAAE,CAAC;YAC3C,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAChE,8CAA8C;YAC9C,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CACnD,IAAI,CAAC,MAAM,CAAC,SAAS,CACtB,CAAC;QACJ,CAAC;QAED,0EAA0E;QAC1E,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,GAAG,IAAI,kBAAkB,CAAC,KAAK,EAAE;YAC7C,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAChC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;SACrC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CACX,IAAY,EACZ,MAA4B;QAE5B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,iBAAiB;QACjB,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAEnD,gBAAgB;QAChB,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAEtE,8BAA8B;QAC9B,MAAM,WAAW,GAAG,aAAa,CAC/B,YAAY,CAAC,MAAM,EACnB,MAAM,EACN,WAAW,EACX,IAAI,CACL,CAAC;QAEF,wDAAwD;QACxD,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,KAAK,GAAG,oBAAoB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;QAE7D,qBAAqB;QACrB,KAAK,GAAG,qBAAqB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC3C,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAExC,oCAAoC;QACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,KAAK,GAAG,KAAK,CAAC,MAAM,CAClB,CAAC,IAAI,EAAE,EAAE,CACP,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBAClC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CACxC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAElC,OAAO;YACL,KAAK;YACL,gBAAgB,EAAE,OAAO,GAAG,SAAS;YACrC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY;SACvC,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CACxB,YAAgC;QAEhC,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAMpB,CAAC;QAEF,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE/C,iBAAiB;QACjB,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CACxC,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EACrD,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,MAAM,mBAAmB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAC7C,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAC1D,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAC5C,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EACzD,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,gBAAgB;QAChB,MAAM,KAAK,GAA4B;YACrC,SAAS,EAAE,cAAc;YACzB,cAAc,EAAE,mBAAmB;SACpC,CAAC;QAEF,uCAAuC;QACvC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACtC,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1C,KAAK,CAAC,gBAAgB,CAAC,GAAG,kBAAkB,CAAC;QAC/C,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEzC,oBAAoB;QACpB,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,+CAA+C;QAC/C,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,aAAa,CACnB,MAA8B,EAC9B,SAAiB;QAEjB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE9C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,4BAA4B;YAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;YACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACjD,CAAC;YAED,gBAAgB;YAChB,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;YAEnC,aAAa;YACb,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC;oBAC9B,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBACxB,MAAM,GAAG,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;YACjD,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,MAA4B;QACnD,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC;QAErC,yCAAyC;QACzC,IAAI,YAAY,GAAG,GAAG,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;YAC/D,IAAI,SAAS,GAAG,YAAY,EAAE,CAAC;gBAC7B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,OAAO;QACL,+EAA+E;QAC/E,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,MAAgB;IAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACpD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,MAA0E;IAE1E,MAAM,UAAU,GAAmB;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,iBAAiB;QAC9C,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,GAAG;QAClC,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,KAAK,EAAE,uBAAuB;QACjE,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,OAAO;KAC7C,CAAC;IAEF,OAAO,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,YAAY;IACd,OAAO,GAAG,YAAY,CAAC;IACvB,MAAM,GAAG,IAAI,CAAC;IAEvB,KAAK,CAAC,IAAI;QACR,QAAQ;IACV,CAAC;IAED,OAAO,CACL,KAAa,EACb,OAA6B;QAE7B,OAAO,OAAO,CAAC,OAAO,CAAC;YACrB,KAAK,EAAE,EAAE;YACT,gBAAgB,EAAE,CAAC;YACnB,YAAY,EAAE,IAAI,CAAC,OAAO;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,QAAQ;QACR,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,OAAO,IAAI,YAAY,EAAE,CAAC;AAC5B,CAAC"}

package/dist/ner/onnx-runtime.d.ts

@@ -0,0 +1,46 @@
+/**
+ * ONNX Runtime Abstraction
+ * Allows switching between onnxruntime-node and onnxruntime-web
+ *
+ * In browsers without a bundler, automatically loads onnxruntime-web from CDN
+ */
+export interface OrtTensor {
+    data: Float32Array | BigInt64Array | Int32Array;
+    dims: readonly number[];
+}
+export interface OrtSession {
+    inputNames: readonly string[];
+    outputNames: readonly string[];
+    run(feeds: Record<string, OrtTensor>): Promise<Record<string, OrtTensor>>;
+}
+export interface OrtInferenceSession {
+    create(pathOrBuffer: string | ArrayBuffer | Uint8Array, options?: unknown): Promise<OrtSession>;
+}
+export interface OrtTensorConstructor {
+    new (type: string, data: Float32Array | BigInt64Array | Int32Array | number[] | bigint[], dims: number[]): OrtTensor;
+}
+export interface OrtRuntime {
+    InferenceSession: OrtInferenceSession;
+    Tensor: OrtTensorConstructor;
+}
+/**
+ * Detects the best ONNX runtime for the current environment
+ */
+export declare function detectRuntime(): "node" | "web";
+/**
+ * Loads the appropriate ONNX runtime
+ */
+export declare function loadRuntime(preferredRuntime?: "node" | "web"): Promise<OrtRuntime>;
+/**
+ * Gets the currently loaded runtime type
+ */
+export declare function getRuntimeType(): "node" | "web" | null;
+/**
+ * Resets the runtime (useful for testing)
+ */
+export declare function resetRuntime(): void;
+declare global {
+    var Bun: unknown;
+    var Deno: unknown;
+}
+//# sourceMappingURL=onnx-runtime.d.ts.map

package/dist/ner/onnx-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"onnx-runtime.d.ts","sourceRoot":"","sources":["../../src/ner/onnx-runtime.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,UAAU,CAAC;IAChD,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9B,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IAC/B,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;CAC3E;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,CACJ,YAAY,EAAE,MAAM,GAAG,WAAW,GAAG,UAAU,EAC/C,OAAO,CAAC,EAAE,OAAO,GAChB,OAAO,CAAC,UAAU,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,KACE,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,UAAU,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,EACrE,IAAI,EAAE,MAAM,EAAE,GACb,SAAS,CAAC;CACd;AAED,MAAM,WAAW,UAAU;IACzB,gBAAgB,EAAE,mBAAmB,CAAC;IACtC,MAAM,EAAE,oBAAoB,CAAC;CAC9B;AAQD;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,GAAG,KAAK,CA4B9C;AAiCD;;GAEG;AACH,wBAAsB,WAAW,CAC/B,gBAAgB,CAAC,EAAE,MAAM,GAAG,KAAK,GAChC,OAAO,CAAC,UAAU,CAAC,CA2CrB;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,MAAM,GAAG,KAAK,GAAG,IAAI,CAEtD;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,IAAI,CAGnC;AAGD,OAAO,CAAC,MAAM,CAAC;IAEb,IAAI,GAAG,EAAE,OAAO,CAAC;IAEjB,IAAI,IAAI,EAAE,OAAO,CAAC;CACnB"}

package/dist/ner/onnx-runtime.js

@@ -0,0 +1,130 @@
+/**
+ * ONNX Runtime Abstraction
+ * Allows switching between onnxruntime-node and onnxruntime-web
+ *
+ * In browsers without a bundler, automatically loads onnxruntime-web from CDN
+ */
+// CDN URL for onnxruntime-web (used when bare import fails in browser)
+// Using the bundled ESM version that includes WebAssembly backend
+const ONNX_WEB_CDN_URL = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.19.2/dist/ort.bundle.min.mjs";
+/**
+ * Runtime detection and loading
+ */
+let _runtime = null;
+let _runtimeType = null;
+/**
+ * Detects the best ONNX runtime for the current environment
+ */
+export function detectRuntime() {
+    // Check if we're in Bun
+    const isBun = typeof globalThis.Bun !== "undefined";
+    // Check if we're in a browser-like environment
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+    const isBrowser = typeof globalThis.window !== "undefined";
+    // Check if we're in Deno
+    const isDeno = typeof globalThis.Deno !== "undefined";
+    if (isBrowser || isDeno) {
+        return "web";
+    }
+    // For Bun, try node first, fall back to web
+    if (isBun) {
+        try {
+            // Quick check if onnxruntime-node is loadable
+            require.resolve("onnxruntime-node");
+            return "node";
+        }
+        catch {
+            return "web";
+        }
+    }
+    // Default to node for Node.js
+    return "node";
+}
+/**
+ * Attempts to load onnxruntime-web, first via bare import, then via CDN
+ */
+async function loadOnnxWeb() {
+    // First try bare import (works with bundlers or import maps)
+    try {
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore - onnxruntime-web may not be installed
+        const ort = (await import("onnxruntime-web"));
+        return ort;
+    }
+    catch {
+        // Bare import failed, try CDN (for browsers without bundlers)
+        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+        const isBrowser = typeof globalThis.window !== "undefined";
+        if (isBrowser) {
+            try {
+                // Dynamic import from CDN URL
+                const ort = (await import(
+                /* webpackIgnore: true */ ONNX_WEB_CDN_URL));
+                return ort;
+            }
+            catch (cdnError) {
+                throw new Error(`Failed to load onnxruntime-web from CDN: ${String(cdnError)}`);
+            }
+        }
+        throw new Error("onnxruntime-web is not available");
+    }
+}
+/**
+ * Loads the appropriate ONNX runtime
+ */
+export async function loadRuntime(preferredRuntime) {
+    if (_runtime !== null) {
+        return _runtime;
+    }
+    const runtimeType = preferredRuntime ?? detectRuntime();
+    try {
+        if (runtimeType === "node") {
+            // Dynamic import for onnxruntime-node
+            const ort = (await import("onnxruntime-node"));
+            _runtime = ort;
+            _runtimeType = "node";
+        }
+        else {
+            // Load onnxruntime-web (with CDN fallback for browsers)
+            const ort = await loadOnnxWeb();
+            _runtime = ort;
+            _runtimeType = "web";
+        }
+    }
+    catch (e) {
+        // If preferred runtime fails, try the other
+        const fallbackType = runtimeType === "node" ? "web" : "node";
+        try {
+            if (fallbackType === "node") {
+                const ort = (await import("onnxruntime-node"));
+                _runtime = ort;
+                _runtimeType = "node";
+            }
+            else {
+                // Load onnxruntime-web (with CDN fallback for browsers)
+                const ort = await loadOnnxWeb();
+                _runtime = ort;
+                _runtimeType = "web";
+            }
+        }
+        catch {
+            throw new Error(`Failed to load ONNX runtime. Install either 'onnxruntime-node' or 'onnxruntime-web'.\n` +
+                `Original error: ${String(e)}`);
+        }
+    }
+    return _runtime;
+}
+/**
+ * Gets the currently loaded runtime type
+ */
+export function getRuntimeType() {
+    return _runtimeType;
+}
+/**
+ * Resets the runtime (useful for testing)
+ */
+export function resetRuntime() {
+    _runtime = null;
+    _runtimeType = null;
+}
+//# sourceMappingURL=onnx-runtime.js.map

package/dist/ner/onnx-runtime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"onnx-runtime.js","sourceRoot":"","sources":["../../src/ner/onnx-runtime.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,uEAAuE;AACvE,kEAAkE;AAClE,MAAM,gBAAgB,GACpB,6EAA6E,CAAC;AAkChF;;GAEG;AACH,IAAI,QAAQ,GAAsB,IAAI,CAAC;AACvC,IAAI,YAAY,GAA0B,IAAI,CAAC;AAE/C;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,wBAAwB;IACxB,MAAM,KAAK,GAAG,OAAO,UAAU,CAAC,GAAG,KAAK,WAAW,CAAC;IAEpD,+CAA+C;IAC/C,uEAAuE;IACvE,MAAM,SAAS,GAAG,OAAO,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC;IAE3D,yBAAyB;IACzB,MAAM,MAAM,GAAG,OAAO,UAAU,CAAC,IAAI,KAAK,WAAW,CAAC;IAEtD,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,4CAA4C;IAC5C,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC;YACH,8CAA8C;YAC9C,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YACpC,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW;IACxB,6DAA6D;IAC7D,IAAI,CAAC;QACH,6DAA6D;QAC7D,oDAAoD;QACpD,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAe,CAAC;QAC5D,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;QAC9D,uEAAuE;QACvE,MAAM,SAAS,GAAG,OAAO,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC;QAC3D,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC;gBACH,8BAA8B;gBAC9B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM;gBACvB,yBAAyB,CAAC,gBAAgB,CAC3C,CAAe,CAAC;gBACjB,OAAO,GAAG,CAAC;YACb,CAAC;YAAC,OAAO,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,4CAA4C,MAAM,CAAC,QAAQ,CAAC,EAAE,CAC/D,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,gBAAiC;IAEjC,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,WAAW,GAAG,gBAAgB,IAAI,aAAa,EAAE,CAAC;IAExD,IAAI,CAAC;QACH,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YAC3B,sCAAsC;YACtC,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAe,CAAC;YAC7D,QAAQ,GAAG,GAAG,CAAC;YACf,YAAY,GAAG,MAAM,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,wDAAwD;YACxD,MAAM,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;YAChC,QAAQ,GAAG,GAAG,CAAC;YACf,YAAY,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,4CAA4C;QAC5C,MAAM,YAAY,GAAG,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QAE7D,IAAI,CAAC;YACH,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAe,CAAC;gBAC7D,QAAQ,GAAG,GAAG,CAAC;gBACf,YAAY,GAAG,MAAM,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,wDAAwD;gBACxD,MAAM,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;gBAChC,QAAQ,GAAG,GAAG,CAAC;gBACf,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,wFAAwF;gBACtF,mBAAmB,MAAM,CAAC,CAAC,CAAC,EAAE,CACjC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,QAAQ,GAAG,IAAI,CAAC;IAChB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC"}

package/dist/ner/tokenizer.d.ts

@@ -0,0 +1,118 @@
+/**
+ * HuggingFace Tokenizer
+ * Loads and uses tokenizers from HuggingFace's tokenizer.json format
+ * Supports Unigram (SentencePiece) and BPE tokenizers
+ */
+/**
+ * Token with offset information
+ */
+export interface Token {
+    /** Token ID in vocabulary */
+    id: number;
+    /** Token string */
+    token: string;
+    /** Start character offset in original text */
+    start: number;
+    /** End character offset in original text */
+    end: number;
+    /** Whether this is a continuation token */
+    isContinuation: boolean;
+    /** Whether this is a special token */
+    isSpecial: boolean;
+}
+/**
+ * Tokenization result with metadata
+ */
+export interface TokenizationResult {
+    /** Array of tokens */
+    tokens: Token[];
+    /** Input IDs for model */
+    inputIds: number[];
+    /** Attention mask */
+    attentionMask: number[];
+    /** Token type IDs (for BERT-style models) */
+    tokenTypeIds: number[];
+    /** Mapping from token index to character span [start, end] */
+    tokenToCharSpan: Array<[number, number] | null>;
+}
+/**
+ * Tokenizer configuration
+ */
+export interface TokenizerConfig {
+    /** Maximum sequence length */
+    maxLength: number;
+    /** Whether to lowercase input */
+    doLowerCase: boolean;
+}
+/**
+ * Default tokenizer configuration
+ */
+export declare const DEFAULT_TOKENIZER_CONFIG: TokenizerConfig;
+/**
+ * WordPiece Tokenizer - supports both HuggingFace JSON and vocab.txt formats
+ */
+export declare class WordPieceTokenizer {
+    private vocab;
+    private inverseVocab;
+    private config;
+    private sortedVocab;
+    private clsId;
+    private sepId;
+    private padId;
+    private unkId;
+    private clsToken;
+    private sepToken;
+    private padToken;
+    private unkToken;
+    constructor(vocab: Map<string, number>, config?: Partial<TokenizerConfig>);
+    /**
+     * Detect special tokens from vocabulary
+     */
+    private detectSpecialTokens;
+    /**
+     * Tokenizes text into tokens with offset tracking
+     */
+    tokenize(text: string): TokenizationResult;
+    /**
+     * Find the best matching token using greedy longest-match
+     */
+    private findBestToken;
+    /**
+     * Decodes token IDs back to text
+     */
+    decode(tokenIds: number[]): string;
+    /**
+     * Gets vocabulary size
+     */
+    get vocabSize(): number;
+    /**
+     * Gets a token ID by string
+     */
+    getTokenId(token: string): number | undefined;
+    /**
+     * Gets a token string by ID
+     */
+    getToken(id: number): string | undefined;
+}
+/**
+ * Loads vocabulary from a file (supports tokenizer.json and vocab.txt)
+ * Uses storage abstraction for browser compatibility
+ */
+export declare function loadVocabFromFile(filePath: string): Promise<Map<string, number>>;
+/**
+ * Loads vocabulary from content string (for when content is already available)
+ */
+export declare function loadVocabFromContent(content: string, format?: 'json' | 'txt'): Map<string, number>;
+/**
+ * Parses HuggingFace tokenizer.json format
+ */
+export declare function parseHFTokenizerJson(content: string): Map<string, number>;
+/**
+ * Parses vocabulary from string content (vocab.txt format)
+ */
+export declare function parseVocab(content: string): Map<string, number>;
+/**
+ * Creates a minimal vocabulary for testing
+ */
+export declare function createTestVocab(): Map<string, number>;
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/ner/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/ner/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,mBAAmB;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,4CAA4C;IAC5C,GAAG,EAAE,MAAM,CAAC;IACZ,2CAA2C;IAC3C,cAAc,EAAE,OAAO,CAAC;IACxB,sCAAsC;IACtC,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,sBAAsB;IACtB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,qBAAqB;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8DAA8D;IAC9D,eAAe,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;CACjD;AAsBD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,iCAAiC;IACjC,WAAW,EAAE,OAAO,CAAC;CACtB;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAGtC,CAAC;AAEF;;GAEG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,WAAW,CAA0B;IAG7C,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAG1B,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAE,OAAO,CAAC,eAAe,CAAM;IAiB7E;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,kBAAkB;IAsF1C;;OAEG;IACH,OAAO,CAAC,aAAa;IA0CrB;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;IAmBlC;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAI7C;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;CAGzC;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAWtF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,GAAE,MAAM,GAAG,KAAc,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAM1G;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAmCzE;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAY/D;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAuBrD"}