rehydra 0.1.0

package/dist/index.js

@@ -0,0 +1,384 @@
+/**
+ * Rehydra Module
+ * Main entry point for on-device PII anonymization
+ */
+// Re-export types
+export * from "./types/index.js";
+// Re-export recognizers
+export { RegexRecognizer, RecognizerRegistry, createDefaultRegistry, createRegistry, getGlobalRegistry, emailRecognizer, phoneRecognizer, ibanRecognizer, bicSwiftRecognizer, creditCardRecognizer, ipAddressRecognizer, urlRecognizer, createCustomIdRecognizer, createCaseIdRecognizer, createCustomerIdRecognizer, } from "./recognizers/index.js";
+// Re-export NER components
+export { NERModel, NERModelStub, createNERModel, createNERModelStub, WordPieceTokenizer, loadVocabFromFile, parseVocab, loadRuntime, detectRuntime, getRuntimeType, MODEL_REGISTRY, getModelCacheDir, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, } from "./ner/index.js";
+// Re-export pipeline components
+export { prenormalize, resolveEntities, tagEntities, validateOutput, generateTag, parseTag, rehydrate, enrichSemantics, inferGender, classifyLocation, getDatabaseStats, hasName, hasLocation, 
+// Semantic data loader exports
+isSemanticDataAvailable, isSemanticDataDownloaded, getSemanticDataCacheDir, getDataDirectory, downloadSemanticData, ensureSemanticData, initializeSemanticData, loadSemanticData, clearSemanticData, clearSemanticDataCache, getSemanticDataInfo, SEMANTIC_DATA_FILES, 
+// Title extractor exports
+extractTitle, extractTitlesFromSpans, mergeAdjacentTitleSpans, getTitlesForLanguage, getAllTitles, startsWithTitle, isOnlyTitle, } from "./pipeline/index.js";
+// Re-export crypto
+export { encryptPIIMap, decryptPIIMap, generateKey, deriveKey, generateSalt, InMemoryKeyProvider, ConfigKeyProvider, validateKey, secureCompare, uint8ArrayToBase64, base64ToUint8Array, } from "./crypto/index.js";
+// Re-export storage utilities
+export { getStorageProvider, isNode, isBrowser, resetStorageProvider, setStorageProvider, } from "./utils/storage.js";
+// Re-export path utilities
+export { join as pathJoin, dirname as pathDirname, basename as pathBasename, normalize as pathNormalize, extname as pathExtname, isAbsolute as pathIsAbsolute, } from "./utils/path.js";
+// Main anonymization imports
+import { createDefaultPolicy, } from "./types/index.js";
+/**
+ * Merges a partial policy with a base policy (deep merge for Maps/Sets)
+ * Unlike the exported mergePolicy, this uses a custom base instead of global defaults
+ */
+function mergePolicyWithBase(base, partial) {
+    // Deep merge confidenceThresholds Map
+    let confidenceThresholds = base.confidenceThresholds;
+    if (partial.confidenceThresholds !== undefined) {
+        confidenceThresholds = new Map(base.confidenceThresholds);
+        for (const [type, threshold] of partial.confidenceThresholds) {
+            confidenceThresholds.set(type, threshold);
+        }
+    }
+    return {
+        enabledTypes: partial.enabledTypes ?? base.enabledTypes,
+        regexEnabledTypes: partial.regexEnabledTypes ?? base.regexEnabledTypes,
+        nerEnabledTypes: partial.nerEnabledTypes ?? base.nerEnabledTypes,
+        typePriority: partial.typePriority ?? base.typePriority,
+        confidenceThresholds,
+        customIdPatterns: partial.customIdPatterns ?? base.customIdPatterns,
+        allowlistTerms: partial.allowlistTerms ?? base.allowlistTerms,
+        denylistPatterns: partial.denylistPatterns ?? base.denylistPatterns,
+        reuseIdsForRepeatedPII: partial.reuseIdsForRepeatedPII ?? base.reuseIdsForRepeatedPII,
+        enableLeakScan: partial.enableLeakScan ?? base.enableLeakScan,
+        enableSemanticMasking: partial.enableSemanticMasking ?? base.enableSemanticMasking,
+    };
+}
+import { createDefaultRegistry, } from "./recognizers/index.js";
+import { NERModelStub, createNERModel, DEFAULT_LABEL_MAP, } from "./ner/index.js";
+import { ensureModel, } from "./ner/model-manager.js";
+import { prenormalize } from "./pipeline/prenormalize.js";
+import { resolveEntities } from "./pipeline/resolver.js";
+import { tagEntities, countEntitiesByType } from "./pipeline/tagger.js";
+import { validateOutput } from "./pipeline/validator.js";
+import { enrichSemantics } from "./pipeline/semantic-enricher.js";
+import { ensureSemanticData, isSemanticDataAvailable, loadSemanticData, } from "./pipeline/semantic-data-loader.js";
+import { extractTitlesFromSpans, mergeAdjacentTitleSpans, } from "./pipeline/title-extractor.js";
+import { encryptPIIMap, generateKey, } from "./crypto/index.js";
+import { getStorageProvider } from "./utils/storage.js";
+/**
+ * Anonymizer instance
+ * Main class for performing PII anonymization
+ */
+export class Anonymizer {
+    registry;
+    nerModel = null;
+    nerConfig;
+    semanticConfig;
+    keyProvider;
+    defaultPolicy;
+    modelVersion;
+    policyVersion;
+    initialized = false;
+    semanticDataReady = false;
+    constructor(config = {}) {
+        this.registry = config.registry ?? createDefaultRegistry();
+        this.keyProvider = config.keyProvider ?? null;
+        this.defaultPolicy = config.defaultPolicy ?? createDefaultPolicy();
+        this.policyVersion = config.policyVersion ?? "1.0.0";
+        // Handle NER configuration
+        this.nerConfig = config.ner ?? { mode: "disabled" };
+        this.modelVersion = config.modelVersion ?? "1.0.0";
+        // Merge NER thresholds into default policy if provided
+        if (this.nerConfig.thresholds !== undefined) {
+            const thresholdsMap = new Map(this.defaultPolicy.confidenceThresholds);
+            for (const [type, threshold] of Object.entries(this.nerConfig.thresholds)) {
+                if (threshold !== undefined) {
+                    thresholdsMap.set(type, threshold);
+                }
+            }
+            this.defaultPolicy = {
+                ...this.defaultPolicy,
+                confidenceThresholds: thresholdsMap,
+            };
+        }
+        // Handle semantic configuration
+        this.semanticConfig = config.semantic ?? { enabled: false };
+        // If semantic is enabled, also enable it in the default policy
+        if (this.semanticConfig.enabled) {
+            this.defaultPolicy = {
+                ...this.defaultPolicy,
+                enableSemanticMasking: true,
+            };
+        }
+    }
+    /**
+     * Initializes the anonymizer
+     * Downloads NER model and semantic data if needed and loads them
+     */
+    async initialize() {
+        if (this.initialized)
+            return;
+        // Handle NER model setup based on mode
+        if (this.nerConfig.mode === "disabled") {
+            this.nerModel = new NERModelStub();
+        }
+        else if (this.nerConfig.mode === "custom") {
+            if (this.nerConfig.modelPath === undefined ||
+                this.nerConfig.modelPath === "" ||
+                this.nerConfig.vocabPath === undefined ||
+                this.nerConfig.vocabPath === "") {
+                throw new Error("NER mode 'custom' requires modelPath and vocabPath");
+            }
+            this.nerModel = createNERModel({
+                modelPath: this.nerConfig.modelPath,
+                vocabPath: this.nerConfig.vocabPath,
+                modelVersion: this.modelVersion,
+            });
+        }
+        else {
+            // 'standard' or 'quantized' - use model manager
+            const { modelPath, vocabPath, labelMapPath } = await ensureModel(this.nerConfig.mode, {
+                autoDownload: this.nerConfig.autoDownload ?? true,
+                onProgress: this.nerConfig.onDownloadProgress,
+                onStatus: this.nerConfig.onStatus,
+            });
+            // Load label map
+            let labelMap = DEFAULT_LABEL_MAP;
+            try {
+                const storage = await getStorageProvider();
+                const labelMapContent = await storage.readTextFile(labelMapPath);
+                labelMap = JSON.parse(labelMapContent);
+            }
+            catch {
+                // Use default label map
+            }
+            this.nerModel = createNERModel({
+                modelPath,
+                vocabPath,
+                labelMap,
+                modelVersion: this.modelVersion,
+            });
+        }
+        // Load the NER model
+        if (!this.nerModel.loaded) {
+            this.nerConfig.onStatus?.("Loading NER model...");
+            await this.nerModel.load();
+            this.nerConfig.onStatus?.("NER model loaded!");
+        }
+        // Handle semantic data setup if enabled
+        if (this.semanticConfig.enabled) {
+            const autoDownload = this.semanticConfig.autoDownload ?? true;
+            // Check if data is already available
+            const dataAvailable = await isSemanticDataAvailable();
+            if (!dataAvailable) {
+                if (!autoDownload) {
+                    throw new Error("Semantic masking is enabled but data files are not available.\n\n" +
+                        "To download automatically, use:\n" +
+                        "  createAnonymizer({ semantic: { enabled: true, autoDownload: true } })\n\n" +
+                        "Or disable semantic masking:\n" +
+                        "  createAnonymizer({ semantic: { enabled: false } })");
+                }
+                // Download semantic data
+                await ensureSemanticData({
+                    autoDownload: true,
+                    onProgress: this.semanticConfig.onDownloadProgress,
+                    onStatus: this.semanticConfig.onStatus,
+                });
+            }
+            else {
+                this.semanticConfig.onStatus?.("Semantic data already cached");
+            }
+            // Load data into memory for synchronous access during enrichment
+            await loadSemanticData();
+            this.semanticDataReady = true;
+        }
+        this.modelVersion = this.nerModel.version;
+        this.initialized = true;
+    }
+    /**
+     * Anonymizes text, replacing PII with placeholder tags
+     * @param text - Input text to anonymize
+     * @param locale - Optional locale hint (e.g., 'de-DE', 'en-US')
+     * @param policy - Optional policy override
+     * @returns Anonymization result with anonymized text and encrypted PII map
+     */
+    async anonymize(text, locale, policy) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        const startTime = performance.now();
+        // Merge policy with instance defaults (not global defaults)
+        // This ensures semantic config from constructor is preserved
+        // Uses deep merge for Maps (confidenceThresholds) and Sets
+        const effectivePolicy = policy !== undefined
+            ? mergePolicyWithBase(this.defaultPolicy, policy)
+            : this.defaultPolicy;
+        // Step 1: Pre-normalize text
+        const normalizedText = prenormalize(text);
+        // Step 2: Run regex recognizers
+        const regexMatches = this.registry.findAll(normalizedText, effectivePolicy);
+        // Step 3: Run NER model
+        const nerResult = await this.nerModel.predict(normalizedText, effectivePolicy);
+        const nerMatches = nerResult.spans;
+        // Step 4: Resolve and merge entities
+        const resolvedMatches = resolveEntities(regexMatches, nerMatches, effectivePolicy, normalizedText);
+        // Step 4.5: Merge adjacent title+name PERSON spans (if semantic masking enabled)
+        // This fixes NER models that split "Mrs. Smith" into two entities
+        const mergedMatches = effectivePolicy.enableSemanticMasking === true
+            ? mergeAdjacentTitleSpans(resolvedMatches, normalizedText)
+            : resolvedMatches;
+        // Step 4.6: Extract titles from PERSON entities (if semantic masking enabled)
+        // This strips honorific titles (Dr., Mrs., etc.) so they remain visible for translation
+        const titleExtractedMatches = effectivePolicy.enableSemanticMasking
+            ? extractTitlesFromSpans(mergedMatches, normalizedText)
+            : mergedMatches;
+        // Step 4.6: Enrich with semantic attributes (if enabled)
+        // This adds gender for PERSON and scope for LOCATION entities
+        const enrichedMatches = effectivePolicy.enableSemanticMasking
+            ? enrichSemantics(titleExtractedMatches, {
+                locale: locale !== undefined ? locale.split("-")[0] : undefined, // Extract language code
+            })
+            : titleExtractedMatches;
+        // Step 5: Tag entities and build PII map
+        const { anonymizedText, entities, piiMap } = tagEntities(normalizedText, enrichedMatches, effectivePolicy);
+        // Step 6: Validate output
+        const validation = validateOutput(anonymizedText, entities, Array.from(piiMap.keys()), effectivePolicy);
+        if (!validation.valid) {
+            // Log validation errors (but don't expose raw PII)
+            const safeErrors = validation.errors.map((e) => ({
+                code: e.code,
+                message: e.message,
+            }));
+            // eslint-disable-next-line no-console
+            console.warn("Validation warnings:", safeErrors);
+        }
+        // Step 7: Encrypt PII map
+        const encryptionKey = this.keyProvider !== null
+            ? await this.keyProvider.getKey()
+            : generateKey();
+        const encryptedPiiMap = await encryptPIIMap(piiMap, encryptionKey);
+        // Step 8: Build stats
+        const endTime = performance.now();
+        const stats = {
+            countsByType: countEntitiesByType(entities),
+            totalEntities: entities.length,
+            modelVersion: this.modelVersion,
+            policyVersion: this.policyVersion,
+            processingTimeMs: endTime - startTime,
+            leakScanPassed: validation.leakScanPassed,
+        };
+        // Step 9: Build result (without original text in entities)
+        const safeEntities = entities.map(({ original: _original, ...rest }) => rest);
+        return {
+            anonymizedText,
+            entities: safeEntities,
+            piiMap: encryptedPiiMap,
+            stats,
+        };
+    }
+    /**
+     * Disposes of resources
+     */
+    async dispose() {
+        if (this.nerModel) {
+            await this.nerModel.dispose();
+        }
+        this.initialized = false;
+    }
+    /**
+     * Gets the recognizer registry
+     */
+    getRegistry() {
+        return this.registry;
+    }
+    /**
+     * Gets the NER model
+     */
+    getNERModel() {
+        return this.nerModel;
+    }
+    /**
+     * Whether the anonymizer is initialized
+     */
+    get isInitialized() {
+        return this.initialized;
+    }
+}
+/**
+ * Creates an anonymizer with the specified configuration
+ *
+ * @example
+ * ```typescript
+ * // Regex-only (no NER)
+ * const anonymizer = createAnonymizer();
+ *
+ * // With NER (auto-downloads model on first use)
+ * const anonymizer = createAnonymizer({
+ *   ner: { mode: 'quantized' }
+ * });
+ *
+ * // With NER and progress callback
+ * const anonymizer = createAnonymizer({
+ *   ner: {
+ *     mode: 'standard',
+ *     onStatus: (status) => console.log(status),
+ *     onDownloadProgress: (p) => console.log(`${p.file}: ${p.percent}%`)
+ *   }
+ * });
+ * ```
+ */
+export function createAnonymizer(config) {
+    return new Anonymizer(config);
+}
+/**
+ * Convenience function for one-off anonymization
+ * Creates a temporary anonymizer with default settings (regex-only)
+ */
+export async function anonymize(text, locale, policy) {
+    const anonymizer = createAnonymizer();
+    await anonymizer.initialize();
+    try {
+        return await anonymizer.anonymize(text, locale, policy);
+    }
+    finally {
+        await anonymizer.dispose();
+    }
+}
+/**
+ * Quick regex-only anonymization (no NER, faster)
+ */
+export async function anonymizeRegexOnly(text, policy) {
+    // Create policy with NER disabled
+    const regexOnlyPolicy = {
+        ...policy,
+        nerEnabledTypes: new Set(), // Disable all NER types
+    };
+    return anonymize(text, undefined, regexOnlyPolicy);
+}
+/**
+ * Full anonymization with NER
+ * Auto-downloads the quantized model on first use
+ *
+ * @example
+ * ```typescript
+ * const result = await anonymizeWithNER(
+ *   'Contact John Smith at john@example.com',
+ *   {
+ *     mode: 'quantized',
+ *     onStatus: console.log
+ *   }
+ * );
+ * ```
+ */
+export async function anonymizeWithNER(text, nerConfig, policy) {
+    const anonymizer = createAnonymizer({
+        ner: {
+            mode: nerConfig.mode ?? "quantized",
+            ...nerConfig,
+        },
+    });
+    await anonymizer.initialize();
+    try {
+        return await anonymizer.anonymize(text, undefined, policy);
+    }
+    finally {
+        await anonymizer.dispose();
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,kBAAkB;AAClB,cAAc,kBAAkB,CAAC;AAEjC,wBAAwB;AACxB,OAAO,EAEL,eAAe,EACf,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,wBAAwB,EACxB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAEhC,2BAA2B;AAC3B,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EAMd,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AAExB,gCAAgC;AAChC,OAAO,EACL,YAAY,EACZ,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,OAAO,EACP,WAAW;AACX,+BAA+B;AAC/B,uBAAuB,EACvB,wBAAwB,EACxB,uBAAuB,EACvB,gBAAgB,EAChB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB;AACnB,0BAA0B;AAC1B,YAAY,EACZ,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,YAAY,EACZ,eAAe,EACf,WAAW,GAMZ,MAAM,qBAAqB,CAAC;AAE7B,mBAAmB;AACnB,OAAO,EACL,aAAa,EACb,aAAa,EACb,WAAW,EACX,SAAS,EACT,YAAY,EAEZ,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,aAAa,EACb,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAE3B,8BAA8B;AAC9B,OAAO,EACL,kBAAkB,EAClB,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,kBAAkB,GAEnB,MAAM,oBAAoB,CAAC;AAE5B,2BAA2B;AAC3B,OAAO,EACL,IAAI,IAAI,QAAQ,EAChB,OAAO,IAAI,WAAW,EACtB,QAAQ,IAAI,YAAY,EACxB,SAAS,IAAI,aAAa,EAC1B,OAAO,IAAI,WAAW,EACtB,UAAU,IAAI,cAAc,GAC7B,MAAM,iBAAiB,CAAC;AAEzB,6BAA6B;AAC7B,OAAO,EAQL,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAE1B;;;GAGG;AACH,SAAS,mBAAmB,CAC1B,IAAyB,EACzB,OAAqC;IAErC,sCAAsC;IACtC,IAAI,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC;IACrD,IAAI,OAAO,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;QAC/C,oBAAoB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC1D,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;YAC7D,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,OAAO;QACL,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;QACvD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB;QACtE,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;QAChE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;QACvD,oBAAoB;QACpB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB;QACnE,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc;QAC7D,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB;QACnE,sBAAsB,EACpB,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,sBAAsB;QAC/D,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc;QAC7D,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB;KAC9D,CAAC;AACJ,CAAC;AACD,OAAO,EACL,qBAAqB,GAEtB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,YAAY,EACZ,cAAc,EACd,iBAAiB,GAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAEL,WAAW,GAEZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EACL,kBAAkB,EAClB,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACL,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,aAAa,EACb,WAAW,GAEZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAkFxD;;;GAGG;AACH,MAAM,OAAO,UAAU;IACb,QAAQ,CAAqB;IAC7B,QAAQ,GAAqB,IAAI,CAAC;IAClC,SAAS,CAAY;IACrB,cAAc,CAAiB;IAC/B,WAAW,CAAqB;IAChC,aAAa,CAAsB;IACnC,YAAY,CAAS;IACrB,aAAa,CAAS;IACtB,WAAW,GAAG,KAAK,CAAC;IACpB,iBAAiB,GAAG,KAAK,CAAC;IAElC,YAAY,SAA2B,EAAE;QACvC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC3D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,mBAAmB,EAAE,CAAC;QACnE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,OAAO,CAAC;QAErD,2BAA2B;QAC3B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QACpD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC;QAEnD,uDAAuD;QACvD,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,CAAC;YACvE,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAC5C,IAAI,CAAC,SAAS,CAAC,UAAU,CAC1B,EAAE,CAAC;gBACF,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC5B,aAAa,CAAC,GAAG,CAAC,IAAe,EAAE,SAAS,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;YACD,IAAI,CAAC,aAAa,GAAG;gBACnB,GAAG,IAAI,CAAC,aAAa;gBACrB,oBAAoB,EAAE,aAAa;aACpC,CAAC;QACJ,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAE5D,+DAA+D;QAC/D,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,aAAa,GAAG;gBACnB,GAAG,IAAI,CAAC,aAAa;gBACrB,qBAAqB,EAAE,IAAI;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,uCAAuC;QACvC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACvC,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAY,EAAE,CAAC;QACrC,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5C,IACE,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,SAAS;gBACtC,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,EAAE;gBAC/B,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,SAAS;gBACtC,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,EAAE,EAC/B,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACxE,CAAC;YAED,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;gBACnC,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,MAAM,WAAW,CAC9D,IAAI,CAAC,SAAS,CAAC,IAAI,EACnB;gBACE,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,IAAI,IAAI;gBACjD,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB;gBAC7C,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ;aAClC,CACF,CAAC;YAEF,iBAAiB;YACjB,IAAI,QAAQ,GAAG,iBAAiB,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,kBAAkB,EAAE,CAAC;gBAC3C,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;gBACjE,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAa,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;YAED,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC;gBAC7B,SAAS;gBACT,SAAS;gBACT,QAAQ;gBACR,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC,CAAC;QACL,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,sBAAsB,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,mBAAmB,CAAC,CAAC;QACjD,CAAC;QAED,wCAAwC;QACxC,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,YAAY,IAAI,IAAI,CAAC;YAE9D,qCAAqC;YACrC,MAAM,aAAa,GAAG,MAAM,uBAAuB,EAAE,CAAC;YACtD,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CACb,mEAAmE;wBACjE,mCAAmC;wBACnC,6EAA6E;wBAC7E,gCAAgC;wBAChC,sDAAsD,CACzD,CAAC;gBACJ,CAAC;gBAED,yBAAyB;gBACzB,MAAM,kBAAkB,CAAC;oBACvB,YAAY,EAAE,IAAI;oBAClB,UAAU,EAAE,IAAI,CAAC,cAAc,CAAC,kBAAkB;oBAClD,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;iBACvC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,CAAC,8BAA8B,CAAC,CAAC;YACjE,CAAC;YAED,iEAAiE;YACjE,MAAM,gBAAgB,EAAE,CAAC;YACzB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,MAAe,EACf,MAAqC;QAErC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,4DAA4D;QAC5D,6DAA6D;QAC7D,2DAA2D;QAC3D,MAAM,eAAe,GACnB,MAAM,KAAK,SAAS;YAClB,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;YACjD,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;QAEzB,6BAA6B;QAC7B,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAE1C,gCAAgC;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;QAE5E,wBAAwB;QACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAS,CAAC,OAAO,CAC5C,cAAc,EACd,eAAe,CAChB,CAAC;QACF,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC;QAEnC,qCAAqC;QACrC,MAAM,eAAe,GAAG,eAAe,CACrC,YAAY,EACZ,UAAU,EACV,eAAe,EACf,cAAc,CACf,CAAC;QAEF,iFAAiF;QACjF,kEAAkE;QAClE,MAAM,aAAa,GACjB,eAAe,CAAC,qBAAqB,KAAK,IAAI;YAC5C,CAAC,CAAC,uBAAuB,CAAC,eAAe,EAAE,cAAc,CAAC;YAC1D,CAAC,CAAC,eAAe,CAAC;QAEtB,8EAA8E;QAC9E,wFAAwF;QACxF,MAAM,qBAAqB,GAAG,eAAe,CAAC,qBAAqB;YACjE,CAAC,CAAC,sBAAsB,CAAC,aAAa,EAAE,cAAc,CAAC;YACvD,CAAC,CAAC,aAAa,CAAC;QAElB,yDAAyD;QACzD,8DAA8D;QAC9D,MAAM,eAAe,GAAG,eAAe,CAAC,qBAAqB;YAC3D,CAAC,CAAC,eAAe,CAAC,qBAAqB,EAAE;gBACrC,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,wBAAwB;aAC1F,CAAC;YACJ,CAAC,CAAC,qBAAqB,CAAC;QAE1B,yCAAyC;QACzC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,WAAW,CACtD,cAAc,EACd,eAAe,EACf,eAAe,CAChB,CAAC;QAEF,0BAA0B;QAC1B,MAAM,UAAU,GAAG,cAAc,CAC/B,cAAc,EACd,QAAQ,EACR,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,EACzB,eAAe,CAChB,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YACtB,mDAAmD;YACnD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/C,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC,CAAC;YACJ,sCAAsC;YACtC,OAAO,CAAC,IAAI,CAAC,sBAAsB,EAAE,UAAU,CAAC,CAAC;QACnD,CAAC;QAED,0BAA0B;QAC1B,MAAM,aAAa,GACjB,IAAI,CAAC,WAAW,KAAK,IAAI;YACvB,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;YACjC,CAAC,CAAC,WAAW,EAAE,CAAC;QAEpB,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAEnE,sBAAsB;QACtB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,KAAK,GAAuB;YAChC,YAAY,EAAE,mBAAmB,CAAC,QAAQ,CAAC;YAC3C,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,gBAAgB,EAAE,OAAO,GAAG,SAAS;YACrC,cAAc,EAAE,UAAU,CAAC,cAAc;SAC1C,CAAC;QAEF,2DAA2D;QAC3D,MAAM,YAAY,GAAuC,QAAQ,CAAC,GAAG,CACnE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAC3C,CAAC;QAEF,OAAO;YACL,cAAc;YACd,QAAQ,EAAE,YAAY;YACtB,MAAM,EAAE,eAAe;YACvB,KAAK;SACN,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAyB;IACxD,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,MAAe,EACf,MAAqC;IAErC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IACtC,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,OAAO,MAAM,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1D,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,MAAqC;IAErC,kCAAkC;IAClC,MAAM,eAAe,GAAiC;QACpD,GAAG,MAAM;QACT,eAAe,EAAE,IAAI,GAAG,EAAE,EAAE,wBAAwB;KACrD,CAAC;IAEF,OAAO,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,SAAwE,EACxE,MAAqC;IAErC,MAAM,UAAU,GAAG,gBAAgB,CAAC;QAClC,GAAG,EAAE;YACH,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,WAAW;YACnC,GAAG,SAAS;SACb;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,OAAO,MAAM,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC"}

package/dist/ner/bio-decoder.d.ts

@@ -0,0 +1,64 @@
+/**
+ * BIO Tag Decoder
+ * Converts BIO-tagged token sequences to entity spans
+ */
+import { SpanMatch } from '../types/index.js';
+import type { Token } from './tokenizer.js';
+/**
+ * BIO tag types
+ */
+export declare enum BIOTag {
+    /** Beginning of an entity */
+    B = "B",
+    /** Inside an entity (continuation) */
+    I = "I",
+    /** Outside any entity */
+    O = "O"
+}
+/**
+ * Parsed BIO label
+ */
+export interface ParsedBIOLabel {
+    /** BIO tag type */
+    tag: BIOTag;
+    /** Entity type (null for O tag) */
+    entityType: string | null;
+}
+/**
+ * Raw entity span from NER (before conversion to SpanMatch)
+ */
+export interface RawNEREntity {
+    /** Entity type string from model */
+    type: string;
+    /** Start character offset */
+    start: number;
+    /** End character offset */
+    end: number;
+    /** Combined confidence score */
+    confidence: number;
+    /** Raw text */
+    text: string;
+    /** Token indices that make up this entity */
+    tokenIndices: number[];
+}
+/**
+ * Parses a BIO label string (e.g., "B-PER", "I-ORG", "O")
+ */
+export declare function parseBIOLabel(label: string): ParsedBIOLabel;
+/**
+ * Decodes BIO-tagged tokens into entity spans
+ */
+export declare function decodeBIOTags(tokens: Token[], labels: string[], confidences: number[], originalText: string): RawNEREntity[];
+/**
+ * Converts raw NER entities to SpanMatch format
+ */
+export declare function convertToSpanMatches(rawEntities: RawNEREntity[], confidenceThreshold?: number): SpanMatch[];
+/**
+ * Post-processes NER spans to clean up boundaries
+ */
+export declare function cleanupSpanBoundaries(spans: SpanMatch[], originalText: string): SpanMatch[];
+/**
+ * Merges adjacent spans of the same type
+ */
+export declare function mergeAdjacentSpans(spans: SpanMatch[], originalText: string, maxGap?: number): SpanMatch[];
+//# sourceMappingURL=bio-decoder.d.ts.map

package/dist/ner/bio-decoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bio-decoder.d.ts","sourceRoot":"","sources":["../../src/ner/bio-decoder.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAW,SAAS,EAAmB,MAAM,mBAAmB,CAAC;AAExE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAE5C;;GAEG;AACH,oBAAY,MAAM;IAChB,6BAA6B;IAC7B,CAAC,MAAM;IACP,sCAAsC;IACtC,CAAC,MAAM;IACP,yBAAyB;IACzB,CAAC,MAAM;CACR;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,mCAAmC;IACnC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,oCAAoC;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,2BAA2B;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe;IACf,IAAI,EAAE,MAAM,CAAC;IACb,6CAA6C;IAC7C,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,CAyB3D;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,KAAK,EAAE,EACf,MAAM,EAAE,MAAM,EAAE,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,YAAY,EAAE,MAAM,GACnB,YAAY,EAAE,CAkFhB;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,YAAY,EAAE,EAC3B,mBAAmB,GAAE,MAAY,GAChC,SAAS,EAAE,CA0Bb;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,SAAS,EAAE,EAClB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CAoCb;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,SAAS,EAAE,EAClB,YAAY,EAAE,MAAM,EACpB,MAAM,GAAE,MAAU,GACjB,SAAS,EAAE,CAiCb"}