rehydra 0.1.0

package/dist/ner/bio-decoder.js

@@ -0,0 +1,216 @@
+/**
+ * BIO Tag Decoder
+ * Converts BIO-tagged token sequences to entity spans
+ */
+import { PIIType, DetectionSource } from '../types/index.js';
+import { getPIITypeFromNERLabel } from '../types/pii-types.js';
+/**
+ * BIO tag types
+ */
+export var BIOTag;
+(function (BIOTag) {
+    /** Beginning of an entity */
+    BIOTag["B"] = "B";
+    /** Inside an entity (continuation) */
+    BIOTag["I"] = "I";
+    /** Outside any entity */
+    BIOTag["O"] = "O";
+})(BIOTag || (BIOTag = {}));
+/**
+ * Parses a BIO label string (e.g., "B-PER", "I-ORG", "O")
+ */
+export function parseBIOLabel(label) {
+    if (label === 'O' || label === '[PAD]' || label === '[CLS]' || label === '[SEP]') {
+        return { tag: BIOTag.O, entityType: null };
+    }
+    const parts = label.split('-');
+    if (parts.length !== 2) {
+        return { tag: BIOTag.O, entityType: null };
+    }
+    const [tagStr, entityType] = parts;
+    let tag;
+    switch (tagStr?.toUpperCase()) {
+        case 'B':
+            tag = BIOTag.B;
+            break;
+        case 'I':
+            tag = BIOTag.I;
+            break;
+        default:
+            return { tag: BIOTag.O, entityType: null };
+    }
+    return { tag, entityType: entityType ?? null };
+}
+/**
+ * Decodes BIO-tagged tokens into entity spans
+ */
+export function decodeBIOTags(tokens, labels, confidences, originalText) {
+    const entities = [];
+    let currentEntity = null;
+    for (let i = 0; i < tokens.length; i++) {
+        const token = tokens[i];
+        const label = labels[i] ?? 'O';
+        const confidence = confidences[i] ?? 0;
+        // Skip special tokens
+        if (token.isSpecial) {
+            // If we have a current entity, close it
+            if (currentEntity !== null) {
+                entities.push(currentEntity);
+                currentEntity = null;
+            }
+            continue;
+        }
+        const { tag, entityType } = parseBIOLabel(label);
+        switch (tag) {
+            case BIOTag.B:
+                // Start of new entity
+                // Close previous entity if exists
+                if (currentEntity !== null) {
+                    entities.push(currentEntity);
+                }
+                currentEntity = {
+                    type: entityType ?? 'UNKNOWN',
+                    start: token.start,
+                    end: token.end,
+                    confidence,
+                    text: originalText.slice(token.start, token.end),
+                    tokenIndices: [i],
+                };
+                break;
+            case BIOTag.I:
+                // Continuation of entity
+                if (currentEntity !== null && entityType === currentEntity.type) {
+                    // Extend current entity
+                    currentEntity.end = token.end;
+                    currentEntity.text = originalText.slice(currentEntity.start, currentEntity.end);
+                    currentEntity.tokenIndices.push(i);
+                    // Average confidence
+                    currentEntity.confidence =
+                        (currentEntity.confidence * (currentEntity.tokenIndices.length - 1) + confidence) /
+                            currentEntity.tokenIndices.length;
+                }
+                else {
+                    // I tag without matching B tag - treat as new entity (common in some models)
+                    if (currentEntity !== null) {
+                        entities.push(currentEntity);
+                    }
+                    currentEntity = {
+                        type: entityType ?? 'UNKNOWN',
+                        start: token.start,
+                        end: token.end,
+                        confidence,
+                        text: originalText.slice(token.start, token.end),
+                        tokenIndices: [i],
+                    };
+                }
+                break;
+            case BIOTag.O:
+                // Outside entity - close current if exists
+                if (currentEntity !== null) {
+                    entities.push(currentEntity);
+                    currentEntity = null;
+                }
+                break;
+        }
+    }
+    // Don't forget to close the last entity
+    if (currentEntity !== null) {
+        entities.push(currentEntity);
+    }
+    return entities;
+}
+/**
+ * Converts raw NER entities to SpanMatch format
+ */
+export function convertToSpanMatches(rawEntities, confidenceThreshold = 0.5) {
+    const spans = [];
+    for (const entity of rawEntities) {
+        // Filter by confidence
+        if (entity.confidence < confidenceThreshold) {
+            continue;
+        }
+        // Map entity type to PIIType
+        const piiType = getPIITypeFromNERLabel(entity.type);
+        if (piiType === null) {
+            continue; // Skip unknown types
+        }
+        spans.push({
+            type: piiType,
+            start: entity.start,
+            end: entity.end,
+            confidence: entity.confidence,
+            source: DetectionSource.NER,
+            text: entity.text,
+        });
+    }
+    return spans;
+}
+/**
+ * Post-processes NER spans to clean up boundaries
+ */
+export function cleanupSpanBoundaries(spans, originalText) {
+    return spans.map((span) => {
+        let { start, end } = span;
+        // Trim leading whitespace
+        while (start < end && /\s/.test(originalText[start] ?? '')) {
+            start++;
+        }
+        // Trim trailing whitespace
+        while (end > start && /\s/.test(originalText[end - 1] ?? '')) {
+            end--;
+        }
+        // Trim leading/trailing punctuation for PERSON/ORG types
+        if (span.type === PIIType.PERSON || span.type === PIIType.ORG) {
+            while (start < end && /[.,;:!?'"()]/.test(originalText[start] ?? '')) {
+                start++;
+            }
+            while (end > start && /[.,;:!?'"()]/.test(originalText[end - 1] ?? '')) {
+                end--;
+            }
+        }
+        // If span became empty, return original
+        if (start >= end) {
+            return span;
+        }
+        return {
+            ...span,
+            start,
+            end,
+            text: originalText.slice(start, end),
+        };
+    });
+}
+/**
+ * Merges adjacent spans of the same type
+ */
+export function mergeAdjacentSpans(spans, originalText, maxGap = 1) {
+    if (spans.length <= 1)
+        return spans;
+    // Sort by start position
+    const sorted = [...spans].sort((a, b) => a.start - b.start);
+    const merged = [];
+    let current = sorted[0];
+    for (let i = 1; i < sorted.length; i++) {
+        const next = sorted[i];
+        // Check if same type and close enough
+        const gap = next.start - current.end;
+        const gapText = originalText.slice(current.end, next.start);
+        const isOnlyWhitespace = /^\s*$/.test(gapText);
+        if (next.type === current.type && gap <= maxGap && isOnlyWhitespace) {
+            // Merge spans
+            current = {
+                ...current,
+                end: next.end,
+                text: originalText.slice(current.start, next.end),
+                confidence: (current.confidence + next.confidence) / 2,
+            };
+        }
+        else {
+            merged.push(current);
+            current = next;
+        }
+    }
+    merged.push(current);
+    return merged;
+}
+//# sourceMappingURL=bio-decoder.js.map

package/dist/ner/bio-decoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bio-decoder.js","sourceRoot":"","sources":["../../src/ner/bio-decoder.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAa,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAG/D;;GAEG;AACH,MAAM,CAAN,IAAY,MAOX;AAPD,WAAY,MAAM;IAChB,6BAA6B;IAC7B,iBAAO,CAAA;IACP,sCAAsC;IACtC,iBAAO,CAAA;IACP,yBAAyB;IACzB,iBAAO,CAAA;AACT,CAAC,EAPW,MAAM,KAAN,MAAM,QAOjB;AA8BD;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACjF,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,KAAK,CAAC;IAEnC,IAAI,GAAW,CAAC;IAChB,QAAQ,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;QAC9B,KAAK,GAAG;YACN,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC;YACf,MAAM;QACR,KAAK,GAAG;YACN,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC;YACf,MAAM;QACR;YACE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC/C,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,UAAU,IAAI,IAAI,EAAE,CAAC;AACjD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAe,EACf,MAAgB,EAChB,WAAqB,EACrB,YAAoB;IAEpB,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,IAAI,aAAa,GAAwB,IAAI,CAAC;IAE9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;QAC/B,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEvC,sBAAsB;QACtB,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACpB,wCAAwC;YACxC,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;gBAC3B,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC7B,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC;YACD,SAAS;QACX,CAAC;QAED,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QAEjD,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,CAAC;gBACX,sBAAsB;gBACtB,kCAAkC;gBAClC,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;oBAC3B,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC/B,CAAC;gBAED,aAAa,GAAG;oBACd,IAAI,EAAE,UAAU,IAAI,SAAS;oBAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU;oBACV,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;oBAChD,YAAY,EAAE,CAAC,CAAC,CAAC;iBAClB,CAAC;gBACF,MAAM;YAER,KAAK,MAAM,CAAC,CAAC;gBACX,yBAAyB;gBACzB,IAAI,aAAa,KAAK,IAAI,IAAI,UAAU,KAAK,aAAa,CAAC,IAAI,EAAE,CAAC;oBAChE,wBAAwB;oBACxB,aAAa,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;oBAC9B,aAAa,CAAC,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC;oBAChF,aAAa,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBACnC,qBAAqB;oBACrB,aAAa,CAAC,UAAU;wBACtB,CAAC,aAAa,CAAC,UAAU,GAAG,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;4BACjF,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC;gBACtC,CAAC;qBAAM,CAAC;oBACN,6EAA6E;oBAC7E,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;wBAC3B,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC/B,CAAC;oBACD,aAAa,GAAG;wBACd,IAAI,EAAE,UAAU,IAAI,SAAS;wBAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,GAAG,EAAE,KAAK,CAAC,GAAG;wBACd,UAAU;wBACV,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;wBAChD,YAAY,EAAE,CAAC,CAAC,CAAC;qBAClB,CAAC;gBACJ,CAAC;gBACD,MAAM;YAER,KAAK,MAAM,CAAC,CAAC;gBACX,2CAA2C;gBAC3C,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;oBAC3B,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC7B,aAAa,GAAG,IAAI,CAAC;gBACvB,CAAC;gBACD,MAAM;QACV,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,WAA2B,EAC3B,sBAA8B,GAAG;IAEjC,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QACjC,uBAAuB;QACvB,IAAI,MAAM,CAAC,UAAU,GAAG,mBAAmB,EAAE,CAAC;YAC5C,SAAS;QACX,CAAC;QAED,6BAA6B;QAC7B,MAAM,OAAO,GAAG,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,SAAS,CAAC,qBAAqB;QACjC,CAAC;QAED,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,MAAM,EAAE,eAAe,CAAC,GAAG;YAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAAkB,EAClB,YAAoB;IAEpB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAE1B,0BAA0B;QAC1B,OAAO,KAAK,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAC3D,KAAK,EAAE,CAAC;QACV,CAAC;QAED,2BAA2B;QAC3B,OAAO,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAC7D,GAAG,EAAE,CAAC;QACR,CAAC;QAED,yDAAyD;QACzD,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC;YAC9D,OAAO,KAAK,GAAG,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBACrE,KAAK,EAAE,CAAC;YACV,CAAC;YACD,OAAO,GAAG,GAAG,KAAK,IAAI,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBACvE,GAAG,EAAE,CAAC;YACR,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,GAAG,IAAI;YACP,KAAK;YACL,GAAG;YACH,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;SACrC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,KAAkB,EAClB,YAAoB,EACpB,SAAiB,CAAC;IAElB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEpC,yBAAyB;IACzB,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QAExB,sCAAsC;QACtC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC;QACrC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,MAAM,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE/C,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,IAAI,GAAG,IAAI,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACpE,cAAc;YACd,OAAO,GAAG;gBACR,GAAG,OAAO;gBACV,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;gBACjD,UAAU,EAAE,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;aACvD,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/ner/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * NER Module
+ * Exports NER model and tokenizer components
+ */
+export * from './tokenizer.js';
+export * from './bio-decoder.js';
+export * from './ner-model.js';
+export { loadRuntime, detectRuntime, getRuntimeType } from './onnx-runtime.js';
+export { type NERModelMode, type ModelInfo, type ModelFileInfo, type DownloadProgressCallback, MODEL_REGISTRY, getModelCacheDir, getModelPath, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, getModelInfo, } from './model-manager.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/ner/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ner/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EACL,KAAK,YAAY,EACjB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,EACpB,YAAY,GACb,MAAM,oBAAoB,CAAC"}

package/dist/ner/index.js

@@ -0,0 +1,10 @@
+/**
+ * NER Module
+ * Exports NER model and tokenizer components
+ */
+export * from './tokenizer.js';
+export * from './bio-decoder.js';
+export * from './ner-model.js';
+export { loadRuntime, detectRuntime, getRuntimeType } from './onnx-runtime.js';
+export { MODEL_REGISTRY, getModelCacheDir, getModelPath, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, getModelInfo, } from './model-manager.js';
+//# sourceMappingURL=index.js.map

package/dist/ner/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ner/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EAKL,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,EACpB,YAAY,GACb,MAAM,oBAAoB,CAAC"}

package/dist/ner/model-manager.d.ts

@@ -0,0 +1,111 @@
+/**
+ * NER Model Manager
+ * Handles automatic downloading and caching of NER models from Hugging Face Hub
+ * Browser-compatible using storage abstraction layer
+ */
+/**
+ * Available NER model variants
+ */
+export type NERModelMode = "standard" | "quantized" | "disabled" | "custom";
+/**
+ * Model file info
+ */
+export interface ModelFileInfo {
+    /** Filename in the repo */
+    repoFile: string;
+    /** Local filename */
+    localFile: string;
+    /** Whether file is required */
+    required: boolean;
+}
+/**
+ * Model registry entry
+ */
+export interface ModelInfo {
+    /** Model identifier */
+    id: string;
+    /** Human-readable name */
+    name: string;
+    /** Description */
+    description: string;
+    /** Approximate size */
+    size: string;
+    /** Hugging Face repo ID */
+    hfRepo: string;
+    /** Subfolder in repo (for models with multiple variants) */
+    hfSubfolder?: string;
+    /** Files to download */
+    files: ModelFileInfo[];
+    /** Label map for this model */
+    labelMap: string[];
+}
+/**
+ * Registry of available models hosted on Hugging Face Hub
+ *
+ * Using ELAN's ONNX exports which are optimized for JS/ONNX runtime
+ * https://huggingface.co/tjruesch/xlm-roberta-base-ner-hrl-onnx
+ */
+export declare const MODEL_REGISTRY: Record<"standard" | "quantized", ModelInfo>;
+/**
+ * Gets the cache directory for models
+ * Uses platform-specific cache location (or virtual path in browser)
+ */
+export declare function getModelCacheDir(): Promise<string>;
+/**
+ * Gets the path to a specific model variant
+ */
+export declare function getModelPath(mode: "standard" | "quantized"): Promise<string>;
+/**
+ * Checks if a model is already downloaded
+ */
+export declare function isModelDownloaded(mode: "standard" | "quantized"): Promise<boolean>;
+/**
+ * Progress callback for downloads
+ */
+export type DownloadProgressCallback = (progress: {
+    file: string;
+    bytesDownloaded: number;
+    totalBytes: number | null;
+    percent: number | null;
+}) => void;
+/**
+ * Downloads a model variant from Hugging Face Hub
+ */
+export declare function downloadModel(mode: "standard" | "quantized", onProgress?: DownloadProgressCallback, onStatus?: (status: string) => void): Promise<string>;
+/**
+ * Gets model paths if available, or downloads if needed
+ */
+export declare function ensureModel(mode: "standard" | "quantized", options?: {
+    autoDownload?: boolean;
+    onProgress?: DownloadProgressCallback;
+    onStatus?: (status: string) => void;
+}): Promise<{
+    modelPath: string;
+    vocabPath: string;
+    labelMapPath: string;
+}>;
+/**
+ * Clears cached models
+ */
+export declare function clearModelCache(mode?: "standard" | "quantized"): Promise<void>;
+/**
+ * Lists downloaded models
+ */
+export declare function listDownloadedModels(): Promise<Array<{
+    mode: "standard" | "quantized";
+    path: string;
+    size: string;
+}>>;
+/**
+ * Gets info about available models
+ */
+export declare function getModelInfo(mode: "standard" | "quantized"): ModelInfo;
+/**
+ * Reads a model file as ArrayBuffer (for onnxruntime)
+ */
+export declare function readModelFile(path: string): Promise<ArrayBuffer>;
+/**
+ * Reads a text file from storage
+ */
+export declare function readTextFile(path: string): Promise<string>;
+//# sourceMappingURL=model-manager.d.ts.map

package/dist/ner/model-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-manager.d.ts","sourceRoot":"","sources":["../../src/ner/model-manager.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,qBAAqB;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,uBAAuB;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wBAAwB;IACxB,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,UAAU,GAAG,WAAW,EAAE,SAAS,CAiDtE,CAAC;AAiCF;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAGxD;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,UAAU,GAAG,WAAW,GAC7B,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,UAAU,GAAG,WAAW,GAC7B,OAAO,CAAC,OAAO,CAAC,CAyBlB;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,QAAQ,EAAE;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,KAAK,IAAI,CAAC;AAsGX;;GAEG;AACH,wBAAsB,aAAa,CACjC,IAAI,EAAE,UAAU,GAAG,WAAW,EAC9B,UAAU,CAAC,EAAE,wBAAwB,EACrC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,GAClC,OAAO,CAAC,MAAM,CAAC,CAsDjB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,UAAU,GAAG,WAAW,EAC9B,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,wBAAwB,CAAC;IACtC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC,GACL,OAAO,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC,CAiCzE;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,IAAI,CAAC,EAAE,UAAU,GAAG,WAAW,GAC9B,OAAO,CAAC,IAAI,CAAC,CAUf;AAED;;GAEG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CACnD,KAAK,CAAC;IAAE,IAAI,EAAE,UAAU,GAAG,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CACtE,CAgBA;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,SAAS,CAEtE;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAOtE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAGhE"}