rehydra 0.1.0

package/dist/ner/model-manager.js

@@ -0,0 +1,325 @@
+/**
+ * NER Model Manager
+ * Handles automatic downloading and caching of NER models from Hugging Face Hub
+ * Browser-compatible using storage abstraction layer
+ */
+import { getStorageProvider } from "../utils/storage.js";
+import { join, basename } from "../utils/path.js";
+/**
+ * Registry of available models hosted on Hugging Face Hub
+ *
+ * Using ELAN's ONNX exports which are optimized for JS/ONNX runtime
+ * https://huggingface.co/tjruesch/xlm-roberta-base-ner-hrl-onnx
+ */
+export const MODEL_REGISTRY = {
+    standard: {
+        id: "xlm-roberta-ner-standard",
+        name: "XLM-RoBERTa NER (Standard)",
+        description: "Multilingual NER model supporting EN, DE, FR, ES, and more",
+        size: "~1.1 GB",
+        hfRepo: "tjruesch/xlm-roberta-base-ner-hrl-onnx",
+        hfSubfolder: "onnx",
+        files: [
+            { repoFile: "model.onnx", localFile: "model.onnx", required: true },
+        ],
+        labelMap: [
+            "O",
+            "B-DATE",
+            "I-DATE",
+            "B-PER",
+            "I-PER",
+            "B-ORG",
+            "I-ORG",
+            "B-LOC",
+            "I-LOC",
+        ],
+    },
+    quantized: {
+        id: "xlm-roberta-ner-quantized",
+        name: "XLM-RoBERTa NER (Quantized)",
+        description: "Quantized version, ~4x smaller with minimal accuracy loss",
+        size: "~265 MB",
+        hfRepo: "tjruesch/xlm-roberta-base-ner-hrl-onnx",
+        hfSubfolder: "onnx",
+        files: [
+            {
+                repoFile: "model_quantized.onnx",
+                localFile: "model.onnx",
+                required: true,
+            },
+        ],
+        labelMap: [
+            "O",
+            "B-DATE",
+            "I-DATE",
+            "B-PER",
+            "I-PER",
+            "B-ORG",
+            "I-ORG",
+            "B-LOC",
+            "I-LOC",
+        ],
+    },
+};
+/**
+ * Shared tokenizer files (same for both variants)
+ */
+const TOKENIZER_FILES = [
+    { repoFile: "tokenizer.json", localFile: "tokenizer.json", required: true },
+    {
+        repoFile: "tokenizer_config.json",
+        localFile: "tokenizer_config.json",
+        required: false,
+    },
+    {
+        repoFile: "special_tokens_map.json",
+        localFile: "special_tokens_map.json",
+        required: false,
+    },
+    { repoFile: "config.json", localFile: "config.json", required: false },
+];
+// Cached storage provider
+let storageProvider = null;
+/**
+ * Gets the storage provider (lazily initialized)
+ */
+async function getStorage() {
+    if (storageProvider === null) {
+        storageProvider = await getStorageProvider();
+    }
+    return storageProvider;
+}
+/**
+ * Gets the cache directory for models
+ * Uses platform-specific cache location (or virtual path in browser)
+ */
+export async function getModelCacheDir() {
+    const storage = await getStorage();
+    return storage.getCacheDir("models");
+}
+/**
+ * Gets the path to a specific model variant
+ */
+export async function getModelPath(mode) {
+    const cacheDir = await getModelCacheDir();
+    return join(cacheDir, mode);
+}
+/**
+ * Checks if a model is already downloaded
+ */
+export async function isModelDownloaded(mode) {
+    const storage = await getStorage();
+    const modelDir = await getModelPath(mode);
+    const info = MODEL_REGISTRY[mode];
+    try {
+        // Check if model file exists
+        const modelFile = info.files.find((f) => f.required && f.localFile.includes("model"));
+        if (modelFile) {
+            const modelExists = await storage.exists(join(modelDir, modelFile.localFile));
+            if (!modelExists)
+                return false;
+        }
+        // Check if tokenizer exists
+        const tokenizerExists = await storage.exists(join(modelDir, "tokenizer.json"));
+        return tokenizerExists;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Builds a Hugging Face Hub download URL
+ */
+function getHuggingFaceUrl(repo, filename, subfolder) {
+    const filePath = subfolder !== undefined && subfolder !== ""
+        ? `${subfolder}/${filename}`
+        : filename;
+    return `https://huggingface.co/${repo}/resolve/main/${filePath}`;
+}
+/**
+ * Downloads a file from URL and returns the data
+ */
+async function downloadFileData(url, fileName, onProgress) {
+    const response = await fetch(url, {
+        headers: {
+            "User-Agent": "rehydra/1.0.0",
+        },
+    });
+    if (!response.ok) {
+        if (response.status === 404) {
+            throw new Error(`File not found: ${url}`);
+        }
+        throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`);
+    }
+    const totalBytes = response.headers.get("content-length");
+    const total = totalBytes !== null && totalBytes !== "" ? parseInt(totalBytes, 10) : null;
+    const reader = response.body?.getReader();
+    if (reader === undefined) {
+        throw new Error("Response body is not readable");
+    }
+    const chunks = [];
+    let bytesDownloaded = 0;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const result = await reader.read();
+        if (result.done)
+            break;
+        const value = result.value;
+        chunks.push(value);
+        bytesDownloaded += value.length;
+        if (onProgress) {
+            onProgress({
+                file: fileName,
+                bytesDownloaded,
+                totalBytes: total,
+                percent: total !== null && total > 0
+                    ? Math.round((bytesDownloaded / total) * 100)
+                    : null,
+            });
+        }
+    }
+    // Concatenate all chunks
+    const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(chunk, offset);
+        offset += chunk.length;
+    }
+    return result;
+}
+/**
+ * Downloads a file from URL to storage
+ */
+async function downloadFile(url, destPath, onProgress) {
+    const storage = await getStorage();
+    const fileName = basename(destPath);
+    const data = await downloadFileData(url, fileName, onProgress);
+    await storage.writeFile(destPath, data);
+}
+/**
+ * Downloads a model variant from Hugging Face Hub
+ */
+export async function downloadModel(mode, onProgress, onStatus) {
+    const storage = await getStorage();
+    const info = MODEL_REGISTRY[mode];
+    const modelDir = await getModelPath(mode);
+    // Create directory
+    await storage.mkdir(modelDir);
+    onStatus?.(`Downloading ${info.name} from Hugging Face Hub...`);
+    onStatus?.(`Repository: ${info.hfRepo}`);
+    // Download model files
+    for (const file of info.files) {
+        const url = getHuggingFaceUrl(info.hfRepo, file.repoFile, info.hfSubfolder);
+        const destPath = join(modelDir, file.localFile);
+        onStatus?.(`Downloading ${file.repoFile}...`);
+        try {
+            await downloadFile(url, destPath, onProgress);
+        }
+        catch (e) {
+            if (file.required) {
+                throw new Error(`Failed to download required file ${file.repoFile}: ${String(e)}`);
+            }
+            // Optional files can fail silently
+            onStatus?.(`Skipping optional file ${file.repoFile}`);
+        }
+    }
+    // Download tokenizer files (from repo root, not subfolder)
+    for (const file of TOKENIZER_FILES) {
+        const url = getHuggingFaceUrl(info.hfRepo, file.repoFile);
+        const destPath = join(modelDir, file.localFile);
+        try {
+            await downloadFile(url, destPath, onProgress);
+        }
+        catch (e) {
+            if (file.required) {
+                throw new Error(`Failed to download required file ${file.repoFile}: ${String(e)}`);
+            }
+        }
+    }
+    // Write label map
+    const labelMapPath = join(modelDir, "label_map.json");
+    await storage.writeFile(labelMapPath, JSON.stringify(info.labelMap, null, 2));
+    onStatus?.("Download complete!");
+    return modelDir;
+}
+/**
+ * Gets model paths if available, or downloads if needed
+ */
+export async function ensureModel(mode, options = {}) {
+    const { autoDownload = true, onProgress, onStatus } = options;
+    const modelDir = await getModelPath(mode);
+    const info = MODEL_REGISTRY[mode];
+    // Check if already downloaded
+    const isDownloaded = await isModelDownloaded(mode);
+    if (!isDownloaded) {
+        if (!autoDownload) {
+            throw new Error(`NER model '${mode}' not found at ${modelDir}.\n\n` +
+                `To download automatically, use:\n` +
+                `  createAnonymizer({ ner: { mode: '${mode}', autoDownload: true } })\n\n` +
+                `Or use regex-only mode:\n` +
+                `  createAnonymizer({ ner: { mode: 'disabled' } })`);
+        }
+        await downloadModel(mode, onProgress, onStatus);
+    }
+    else {
+        onStatus?.(`Using cached model: ${info.name}`);
+    }
+    // Find model file
+    const modelFile = info.files.find((f) => f.localFile === "model.onnx");
+    return {
+        modelPath: join(modelDir, modelFile?.localFile ?? "model.onnx"),
+        vocabPath: join(modelDir, "tokenizer.json"),
+        labelMapPath: join(modelDir, "label_map.json"),
+    };
+}
+/**
+ * Clears cached models
+ */
+export async function clearModelCache(mode) {
+    const storage = await getStorage();
+    if (mode) {
+        const modelDir = await getModelPath(mode);
+        await storage.rm(modelDir, { recursive: true, force: true });
+    }
+    else {
+        const cacheDir = await getModelCacheDir();
+        await storage.rm(cacheDir, { recursive: true, force: true });
+    }
+}
+/**
+ * Lists downloaded models
+ */
+export async function listDownloadedModels() {
+    const models = [];
+    for (const mode of ["standard", "quantized"]) {
+        if (await isModelDownloaded(mode)) {
+            const modelPath = await getModelPath(mode);
+            const info = MODEL_REGISTRY[mode];
+            models.push({ mode, path: modelPath, size: info.size });
+        }
+    }
+    return models;
+}
+/**
+ * Gets info about available models
+ */
+export function getModelInfo(mode) {
+    return MODEL_REGISTRY[mode];
+}
+/**
+ * Reads a model file as ArrayBuffer (for onnxruntime)
+ */
+export async function readModelFile(path) {
+    const storage = await getStorage();
+    const data = await storage.readFile(path);
+    return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
+}
+/**
+ * Reads a text file from storage
+ */
+export async function readTextFile(path) {
+    const storage = await getStorage();
+    return storage.readTextFile(path);
+}
+//# sourceMappingURL=model-manager.js.map

package/dist/ner/model-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-manager.js","sourceRoot":"","sources":["../../src/ner/model-manager.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,kBAAkB,EAAwB,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAyClD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GAAgD;IACzE,QAAQ,EAAE;QACR,EAAE,EAAE,0BAA0B;QAC9B,IAAI,EAAE,4BAA4B;QAClC,WAAW,EAAE,4DAA4D;QACzE,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,wCAAwC;QAChD,WAAW,EAAE,MAAM;QACnB,KAAK,EAAE;YACL,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE;SACpE;QACD,QAAQ,EAAE;YACR,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;SACR;KACF;IACD,SAAS,EAAE;QACT,EAAE,EAAE,2BAA2B;QAC/B,IAAI,EAAE,6BAA6B;QACnC,WAAW,EAAE,2DAA2D;QACxE,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,wCAAwC;QAChD,WAAW,EAAE,MAAM;QACnB,KAAK,EAAE;YACL;gBACE,QAAQ,EAAE,sBAAsB;gBAChC,SAAS,EAAE,YAAY;gBACvB,QAAQ,EAAE,IAAI;aACf;SACF;QACD,QAAQ,EAAE;YACR,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;SACR;KACF;CACF,CAAC;AAEF;;GAEG;AACH,MAAM,eAAe,GAAoB;IACvC,EAAE,QAAQ,EAAE,gBAAgB,EAAE,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC3E;QACE,QAAQ,EAAE,uBAAuB;QACjC,SAAS,EAAE,uBAAuB;QAClC,QAAQ,EAAE,KAAK;KAChB;IACD;QACE,QAAQ,EAAE,yBAAyB;QACnC,SAAS,EAAE,yBAAyB;QACpC,QAAQ,EAAE,KAAK;KAChB;IACD,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE;CACvE,CAAC;AAEF,0BAA0B;AAC1B,IAAI,eAAe,GAA2B,IAAI,CAAC;AAEnD;;GAEG;AACH,KAAK,UAAU,UAAU;IACvB,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;QAC7B,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC/C,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,OAAO,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAA8B;IAE9B,MAAM,QAAQ,GAAG,MAAM,gBAAgB,EAAE,CAAC;IAC1C,OAAO,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAA8B;IAE9B,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAElC,IAAI,CAAC;QACH,6BAA6B;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CACnD,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,MAAM,CACtC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,CAAC,CACpC,CAAC;YACF,IAAI,CAAC,WAAW;gBAAE,OAAO,KAAK,CAAC;QACjC,CAAC;QAED,4BAA4B;QAC5B,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,MAAM,CAC1C,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CACjC,CAAC;QACF,OAAO,eAAe,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAYD;;GAEG;AACH,SAAS,iBAAiB,CACxB,IAAY,EACZ,QAAgB,EAChB,SAAkB;IAElB,MAAM,QAAQ,GACZ,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE;QACzC,CAAC,CAAC,GAAG,SAAS,IAAI,QAAQ,EAAE;QAC5B,CAAC,CAAC,QAAQ,CAAC;IACf,OAAO,0BAA0B,IAAI,iBAAiB,QAAQ,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,QAAgB,EAChB,UAAqC;IAErC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,OAAO,EAAE;YACP,YAAY,EAAE,eAAe;SAC9B;KACF,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,IAAI,KAAK,CACb,sBAAsB,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CACvE,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC1D,MAAM,KAAK,GACT,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE7E,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;IAC1C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,iDAAiD;IACjD,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAEnC,IAAI,MAAM,CAAC,IAAI;YAAE,MAAM;QAEvB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAmB,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,eAAe,IAAI,KAAK,CAAC,MAAM,CAAC;QAEhC,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC;gBACT,IAAI,EAAE,QAAQ;gBACd,eAAe;gBACf,UAAU,EAAE,KAAK;gBACjB,OAAO,EACL,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC;oBACzB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;oBAC7C,CAAC,CAAC,IAAI;aACX,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;IAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CACzB,GAAW,EACX,QAAgB,EAChB,UAAqC;IAErC,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEpC,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/D,MAAM,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAA8B,EAC9B,UAAqC,EACrC,QAAmC;IAEnC,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAE1C,mBAAmB;IACnB,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE9B,QAAQ,EAAE,CAAC,eAAe,IAAI,CAAC,IAAI,2BAA2B,CAAC,CAAC;IAChE,QAAQ,EAAE,CAAC,eAAe,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAEzC,uBAAuB;IACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAEhD,QAAQ,EAAE,CAAC,eAAe,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC;QAE9C,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,oCAAoC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAClE,CAAC;YACJ,CAAC;YACD,mCAAmC;YACnC,QAAQ,EAAE,CAAC,0BAA0B,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAEhD,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,oCAAoC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAClE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACtD,MAAM,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE9E,QAAQ,EAAE,CAAC,oBAAoB,CAAC,CAAC;IAEjC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAA8B,EAC9B,UAII,EAAE;IAEN,MAAM,EAAE,YAAY,GAAG,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAE9D,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAElC,8BAA8B;IAC9B,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAEnD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACb,cAAc,IAAI,kBAAkB,QAAQ,OAAO;gBACjD,mCAAmC;gBACnC,sCAAsC,IAAI,gCAAgC;gBAC1E,2BAA2B;gBAC3B,mDAAmD,CACtD,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IAClD,CAAC;SAAM,CAAC;QACN,QAAQ,EAAE,CAAC,uBAAuB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,kBAAkB;IAClB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,YAAY,CAAC,CAAC;IAEvE,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,IAAI,YAAY,CAAC;QAC/D,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC;QAC3C,YAAY,EAAE,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KAC/C,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAA+B;IAE/B,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IAEnC,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,GAAG,MAAM,gBAAgB,EAAE,CAAC;QAC1C,MAAM,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IAGxC,MAAM,MAAM,GAIP,EAAE,CAAC;IAER,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,WAAW,CAAU,EAAE,CAAC;QACtD,IAAI,MAAM,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAA8B;IACzD,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY;IAC9C,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC1C,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CACtB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CACnB,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,OAAO,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC"}

package/dist/ner/ner-model.d.ts

@@ -0,0 +1,114 @@
+/**
+ * NER Model Wrapper
+ * ONNX Runtime integration for Named Entity Recognition
+ * Supports both onnxruntime-node and onnxruntime-web
+ */
+import { SpanMatch, AnonymizationPolicy } from "../types/index.js";
+/**
+ * NER Model configuration
+ */
+export interface NERModelConfig {
+    /** Path to ONNX model file */
+    modelPath: string;
+    /** Path to vocabulary file */
+    vocabPath: string;
+    /** Label mapping (index -> label string) */
+    labelMap: string[];
+    /** Maximum sequence length */
+    maxLength: number;
+    /** Whether model expects lowercase input */
+    doLowerCase: boolean;
+    /** Model version for tracking */
+    modelVersion: string;
+}
+/**
+ * NER prediction result for a single text
+ */
+export interface NERPrediction {
+    /** Detected entity spans */
+    spans: SpanMatch[];
+    /** Processing time in ms */
+    processingTimeMs: number;
+    /** Model version used */
+    modelVersion: string;
+}
+/**
+ * Default label map for common NER models (CoNLL-style)
+ */
+export declare const DEFAULT_LABEL_MAP: string[];
+/**
+ * NER Model wrapper for ONNX inference
+ */
+export declare class NERModel {
+    private ort;
+    private session;
+    private tokenizer;
+    private config;
+    private isLoaded;
+    constructor(config: NERModelConfig);
+    /**
+     * Loads the model and tokenizer
+     */
+    load(): Promise<void>;
+    /**
+     * Predicts entities in text
+     */
+    predict(text: string, policy?: AnonymizationPolicy): Promise<NERPrediction>;
+    /**
+     * Runs ONNX inference
+     */
+    private runInference;
+    /**
+     * Processes model logits to extract labels and confidences
+     */
+    private processLogits;
+    /**
+     * Gets minimum confidence threshold from policy
+     */
+    private getMinConfidence;
+    /**
+     * Gets model version
+     */
+    get version(): string;
+    /**
+     * Checks if model is loaded
+     */
+    get loaded(): boolean;
+    /**
+     * Disposes of model resources
+     */
+    dispose(): Promise<void>;
+}
+/**
+ * Creates a NER model instance with configuration
+ */
+export declare function createNERModel(config: Partial<NERModelConfig> & {
+    modelPath: string;
+    vocabPath: string;
+}): NERModel;
+/**
+ * NER Model stub for when no model is available
+ * Returns empty results - useful for regex-only mode
+ */
+export declare class NERModelStub {
+    readonly version = "stub-1.0.0";
+    readonly loaded = true;
+    load(): Promise<void>;
+    predict(_text: string, _policy?: AnonymizationPolicy): Promise<NERPrediction>;
+    dispose(): Promise<void>;
+}
+/**
+ * Creates a stub NER model (for testing or regex-only mode)
+ */
+export declare function createNERModelStub(): NERModelStub;
+/**
+ * NER model interface for dependency injection
+ */
+export interface INERModel {
+    readonly version: string;
+    readonly loaded: boolean;
+    load(): Promise<void>;
+    predict(text: string, policy?: AnonymizationPolicy): Promise<NERPrediction>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=ner-model.d.ts.map

package/dist/ner/ner-model.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ner-model.d.ts","sourceRoot":"","sources":["../../src/ner/ner-model.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAcnE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,WAAW,EAAE,OAAO,CAAC;IACrB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,4BAA4B;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,UAU7B,CAAC;AAEF;;GAEG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,GAAG,CAA2B;IACtC,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,SAAS,CAAmC;IACpD,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,EAAE,cAAc;IAIlC;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA+B3B;;OAEG;IACG,OAAO,CACX,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC;IA+CzB;;OAEG;YACW,YAAY;IAiE1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAqCrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAexB;;OAEG;IACH,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED;;OAEG;IACH,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAOzB;AAYD;;GAEG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACzE,QAAQ,CAWV;AAED;;;GAGG;AACH,qBAAa,YAAY;IACvB,QAAQ,CAAC,OAAO,gBAAgB;IAChC,QAAQ,CAAC,MAAM,QAAQ;IAEjB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,OAAO,CACL,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,aAAa,CAAC;IAQzB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAIzB;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,YAAY,CAEjD;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC5E,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B"}