rehydra 0.1.0

package/dist/pipeline/resolver.js

@@ -0,0 +1,239 @@
+/**
+ * Entity Resolver
+ * Merges, deduplicates, and resolves overlapping entity detections
+ */
+import { PIIType, DetectionSource, DEFAULT_TYPE_PRIORITY, } from '../types/index.js';
+import { spansOverlap, spanLength, sortSpansByPosition } from '../utils/offsets.js';
+/**
+ * Resolution strategy for overlapping entities
+ */
+export var OverlapStrategy;
+(function (OverlapStrategy) {
+    /** Regex matches always win over NER */
+    OverlapStrategy["REGEX_PRIORITY"] = "REGEX_PRIORITY";
+    /** Longer span wins */
+    OverlapStrategy["LONGER_SPAN"] = "LONGER_SPAN";
+    /** Higher confidence wins */
+    OverlapStrategy["HIGHER_CONFIDENCE"] = "HIGHER_CONFIDENCE";
+    /** Use type priority from policy */
+    OverlapStrategy["TYPE_PRIORITY"] = "TYPE_PRIORITY";
+})(OverlapStrategy || (OverlapStrategy = {}));
+/**
+ * Default resolver configuration
+ */
+export const DEFAULT_RESOLVER_CONFIG = {
+    overlapStrategy: OverlapStrategy.REGEX_PRIORITY,
+    regexPriority: true,
+    minConfidence: 0.5,
+};
+/**
+ * Resolves and merges entity detections from regex and NER
+ */
+export function resolveEntities(regexMatches, nerMatches, policy, originalText, config = {}) {
+    const resolverConfig = { ...DEFAULT_RESOLVER_CONFIG, ...config };
+    // Step 1: Filter by enabled types and confidence thresholds
+    const filteredRegex = filterByPolicy(regexMatches, policy);
+    const filteredNER = filterByPolicy(nerMatches, policy);
+    // Step 2: Apply allowlist filtering
+    const allowlistFilteredRegex = applyAllowlist(filteredRegex, policy, originalText);
+    const allowlistFilteredNER = applyAllowlist(filteredNER, policy, originalText);
+    // Step 3: Combine all matches
+    const allMatches = [...allowlistFilteredRegex, ...allowlistFilteredNER];
+    // Step 4: Remove overlaps based on strategy
+    const resolved = removeOverlaps(allMatches, policy, resolverConfig);
+    // Step 5: Apply denylist patterns (force include)
+    const withDenylist = applyDenylist(resolved, policy, originalText);
+    // Step 6: Final deduplication
+    const deduplicated = deduplicateExact(withDenylist);
+    // Step 7: Sort by position
+    return sortSpansByPosition(deduplicated);
+}
+/**
+ * Filters matches by policy (enabled types and confidence thresholds)
+ */
+function filterByPolicy(matches, policy) {
+    return matches.filter((match) => {
+        // Check if type is enabled
+        if (!policy.enabledTypes.has(match.type)) {
+            return false;
+        }
+        // Check confidence threshold
+        const threshold = policy.confidenceThresholds.get(match.type) ?? 0.5;
+        if (match.confidence < threshold) {
+            return false;
+        }
+        return true;
+    });
+}
+/**
+ * Filters out matches that are in the allowlist (known non-PII terms)
+ */
+function applyAllowlist(matches, policy, _originalText) {
+    if (policy.allowlistTerms.size === 0) {
+        return matches;
+    }
+    return matches.filter((match) => {
+        const matchText = match.text.toLowerCase().trim();
+        return !policy.allowlistTerms.has(matchText);
+    });
+}
+/**
+ * Adds matches from denylist patterns (patterns that must always be PII)
+ */
+function applyDenylist(matches, policy, originalText) {
+    if (policy.denylistPatterns.length === 0) {
+        return matches;
+    }
+    const denylistMatches = [];
+    for (const pattern of policy.denylistPatterns) {
+        const globalPattern = pattern.global
+            ? pattern
+            : new RegExp(pattern.source, pattern.flags + 'g');
+        for (const match of originalText.matchAll(globalPattern)) {
+            if (match.index === undefined)
+                continue;
+            // Check if this is already covered by existing matches
+            const matchIndex = match.index;
+            const alreadyCovered = matches.some((existing) => existing.start <= matchIndex &&
+                existing.end >= matchIndex + match[0].length);
+            if (!alreadyCovered) {
+                denylistMatches.push({
+                    type: PIIType.EMAIL, // Default type for denylist; could be configurable
+                    start: match.index,
+                    end: match.index + match[0].length,
+                    confidence: 1.0,
+                    source: DetectionSource.REGEX,
+                    text: match[0],
+                });
+            }
+        }
+    }
+    return [...matches, ...denylistMatches];
+}
+/**
+ * Removes overlapping spans based on resolution strategy
+ */
+function removeOverlaps(matches, policy, config) {
+    if (matches.length <= 1) {
+        return matches;
+    }
+    // Sort by start position
+    const sorted = sortSpansByPosition(matches);
+    const result = [];
+    for (const match of sorted) {
+        // Find overlapping matches in result
+        const overlappingIdx = result.findIndex((existing) => spansOverlap(match, existing));
+        if (overlappingIdx === -1) {
+            // No overlap, add directly
+            result.push(match);
+        }
+        else {
+            // Has overlap, resolve
+            const existing = result[overlappingIdx];
+            const winner = resolveOverlap(existing, match, policy, config);
+            if (winner === match) {
+                // New match wins, replace existing
+                result[overlappingIdx] = match;
+            }
+            // Otherwise keep existing (do nothing)
+        }
+    }
+    return result;
+}
+/**
+ * Resolves overlap between two spans
+ * Returns the winner
+ */
+function resolveOverlap(a, b, policy, config) {
+    // Rule 1: Regex always beats NER if configured
+    if (config.regexPriority) {
+        if (a.source === DetectionSource.REGEX && b.source !== DetectionSource.REGEX) {
+            return a;
+        }
+        if (b.source === DetectionSource.REGEX && a.source !== DetectionSource.REGEX) {
+            return b;
+        }
+    }
+    // Rule 2: Apply overlap strategy
+    switch (config.overlapStrategy) {
+        case OverlapStrategy.LONGER_SPAN: {
+            const lenA = spanLength(a);
+            const lenB = spanLength(b);
+            if (lenA !== lenB) {
+                return lenA > lenB ? a : b;
+            }
+            break;
+        }
+        case OverlapStrategy.HIGHER_CONFIDENCE: {
+            if (a.confidence !== b.confidence) {
+                return a.confidence > b.confidence ? a : b;
+            }
+            break;
+        }
+        case OverlapStrategy.TYPE_PRIORITY: {
+            const priorityA = getTypePriority(a.type, policy);
+            const priorityB = getTypePriority(b.type, policy);
+            if (priorityA !== priorityB) {
+                return priorityA > priorityB ? a : b;
+            }
+            break;
+        }
+        case OverlapStrategy.REGEX_PRIORITY:
+        default:
+            // Already handled above
+            break;
+    }
+    // Tiebreakers: longer span > higher confidence > type priority
+    const lenA = spanLength(a);
+    const lenB = spanLength(b);
+    if (lenA !== lenB) {
+        return lenA > lenB ? a : b;
+    }
+    if (a.confidence !== b.confidence) {
+        return a.confidence > b.confidence ? a : b;
+    }
+    const priorityA = getTypePriority(a.type, policy);
+    const priorityB = getTypePriority(b.type, policy);
+    if (priorityA !== priorityB) {
+        return priorityA > priorityB ? a : b;
+    }
+    // Final tiebreaker: keep first one
+    return a;
+}
+/**
+ * Gets type priority from policy (higher = more important)
+ */
+function getTypePriority(type, policy) {
+    const priorityList = policy.typePriority.length > 0 ? policy.typePriority : [...DEFAULT_TYPE_PRIORITY];
+    const index = priorityList.indexOf(type);
+    return index >= 0 ? index : -1;
+}
+/**
+ * Removes exact duplicate spans
+ */
+function deduplicateExact(matches) {
+    const seen = new Set();
+    const result = [];
+    for (const match of matches) {
+        const key = `${match.start}:${match.end}:${match.type}`;
+        if (!seen.has(key)) {
+            seen.add(key);
+            result.push(match);
+        }
+    }
+    return result;
+}
+/**
+ * Creates protected spans from regex matches
+ * Used to mask regex matches from NER to avoid double-detection
+ */
+export function createProtectedSpans(regexMatches) {
+    return regexMatches.map(({ start, end }) => ({ start, end }));
+}
+/**
+ * Checks if a span overlaps with any protected span
+ */
+export function isInProtectedSpan(span, protectedSpans) {
+    return protectedSpans.some((protected_) => spansOverlap(span, protected_));
+}
+//# sourceMappingURL=resolver.js.map

package/dist/pipeline/resolver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolver.js","sourceRoot":"","sources":["../../src/pipeline/resolver.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,OAAO,EAEP,eAAe,EAEf,qBAAqB,GACtB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAEpF;;GAEG;AACH,MAAM,CAAN,IAAY,eASX;AATD,WAAY,eAAe;IACzB,wCAAwC;IACxC,oDAAiC,CAAA;IACjC,uBAAuB;IACvB,8CAA2B,CAAA;IAC3B,6BAA6B;IAC7B,0DAAuC,CAAA;IACvC,oCAAoC;IACpC,kDAA+B,CAAA;AACjC,CAAC,EATW,eAAe,KAAf,eAAe,QAS1B;AAcD;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAmB;IACrD,eAAe,EAAE,eAAe,CAAC,cAAc;IAC/C,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,GAAG;CACnB,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,YAAyB,EACzB,UAAuB,EACvB,MAA2B,EAC3B,YAAoB,EACpB,SAAkC,EAAE;IAEpC,MAAM,cAAc,GAAG,EAAE,GAAG,uBAAuB,EAAE,GAAG,MAAM,EAAE,CAAC;IAEjE,4DAA4D;IAC5D,MAAM,aAAa,GAAG,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,WAAW,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAEvD,oCAAoC;IACpC,MAAM,sBAAsB,GAAG,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IACnF,MAAM,oBAAoB,GAAG,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAE/E,8BAA8B;IAC9B,MAAM,UAAU,GAAG,CAAC,GAAG,sBAAsB,EAAE,GAAG,oBAAoB,CAAC,CAAC;IAExE,4CAA4C;IAC5C,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,cAAc,CAAC,CAAC;IAEpE,kDAAkD;IAClD,MAAM,YAAY,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAEnE,8BAA8B;IAC9B,MAAM,YAAY,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAEpD,2BAA2B;IAC3B,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,OAAoB,EAAE,MAA2B;IACvE,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC9B,2BAA2B;QAC3B,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,6BAA6B;QAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;QACrE,IAAI,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CACrB,OAAoB,EACpB,MAA2B,EAC3B,aAAqB;IAErB,IAAI,MAAM,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACrC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC9B,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAClD,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CACpB,OAAoB,EACpB,MAA2B,EAC3B,YAAoB;IAEpB,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,eAAe,GAAgB,EAAE,CAAC;IAExC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM;YAClC,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;QAEpD,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACzD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,SAAS;YAExC,uDAAuD;YACvD,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC;YAC/B,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CACjC,CAAC,QAAQ,EAAE,EAAE,CACX,QAAQ,CAAC,KAAK,IAAI,UAAU;gBAC5B,QAAQ,CAAC,GAAG,IAAI,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAC/C,CAAC;YAEF,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,eAAe,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,OAAO,CAAC,KAAK,EAAE,mDAAmD;oBACxE,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;oBAClC,UAAU,EAAE,GAAG;oBACf,MAAM,EAAE,eAAe,CAAC,KAAK;oBAC7B,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,OAAO,EAAE,GAAG,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CACrB,OAAoB,EACpB,MAA2B,EAC3B,MAAsB;IAEtB,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,yBAAyB;IACzB,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,qCAAqC;QACrC,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QAErF,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;YAC1B,2BAA2B;YAC3B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,uBAAuB;YACvB,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAE,CAAC;YACzC,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YAE/D,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;gBACrB,mCAAmC;gBACnC,MAAM,CAAC,cAAc,CAAC,GAAG,KAAK,CAAC;YACjC,CAAC;YACD,uCAAuC;QACzC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CACrB,CAAY,EACZ,CAAY,EACZ,MAA2B,EAC3B,MAAsB;IAEtB,+CAA+C;IAC/C,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,MAAM,KAAK,eAAe,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,KAAK,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7E,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,eAAe,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,KAAK,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7E,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAC/B,KAAK,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC;YACjC,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC3B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClB,OAAO,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;YACD,MAAM;QACR,CAAC;QAED,KAAK,eAAe,CAAC,iBAAiB,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;gBAClC,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7C,CAAC;YACD,MAAM;QACR,CAAC;QAED,KAAK,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC;YACnC,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClD,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,MAAM;QACR,CAAC;QAED,KAAK,eAAe,CAAC,cAAc,CAAC;QACpC;YACE,wBAAwB;YACxB,MAAM;IACV,CAAC;IAED,+DAA+D;IAC/D,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC3B,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;QAClC,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,mCAAmC;IACnC,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,IAAa,EAAE,MAA2B;IACjE,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,qBAAqB,CAAC,CAAC;IACvG,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAoB;IAC5C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,YAAyB;IAEzB,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAC/B,IAAoC,EACpC,cAAqD;IAErD,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;AAC7E,CAAC"}

package/dist/pipeline/semantic-data-loader.d.ts

@@ -0,0 +1,165 @@
+/**
+ * Semantic Data Loader
+ * Handles automatic downloading, caching, and parsing of semantic enrichment data.
+ * Browser-compatible using storage abstraction layer
+ *
+ * Data sources:
+ * - nam_dict.txt: Name-gender mappings from gender-guesser (~40K names)
+ * - cities15000.txt: GeoNames cities with population > 15,000 (~25K cities)
+ * - countryInfo.txt: Country names and codes (~250 countries)
+ * - admin1CodesASCII.txt: First-level admin divisions (~4K regions)
+ *
+ * Data is cached using the storage abstraction layer:
+ * - Node.js: Platform-specific cache directories
+ * - Browser: IndexedDB
+ */
+/**
+ * Loaded name-gender data
+ */
+interface NameEntry {
+    gender: string;
+    localeOverrides?: Record<string, string>;
+}
+/**
+ * Loaded location data
+ */
+interface CityEntry {
+    country: string;
+    population: number;
+}
+interface RegionEntry {
+    country: string;
+    name: string;
+}
+/**
+ * Semantic data store
+ */
+interface SemanticData {
+    names: Map<string, NameEntry>;
+    cities: Map<string, CityEntry>;
+    countries: Map<string, string>;
+    regions: Map<string, RegionEntry>;
+    loaded: boolean;
+}
+/**
+ * Gets the cache directory for semantic data
+ * Uses platform-specific cache location (or virtual path in browser)
+ */
+export declare function getSemanticDataCacheDir(): Promise<string>;
+/**
+ * Gets the path to the data directory (alias for backwards compatibility)
+ */
+export declare function getDataDirectory(): Promise<string>;
+/**
+ * Semantic data file info
+ */
+export interface SemanticDataFileInfo {
+    /** Filename */
+    filename: string;
+    /** Download URL */
+    url: string;
+    /** Whether file is required */
+    required: boolean;
+    /** Description */
+    description: string;
+    /** Approximate size */
+    size: string;
+}
+/**
+ * Registry of semantic data files and their download URLs
+ */
+export declare const SEMANTIC_DATA_FILES: SemanticDataFileInfo[];
+/**
+ * Progress callback for downloads
+ */
+export type SemanticDownloadProgressCallback = (progress: {
+    file: string;
+    bytesDownloaded: number;
+    totalBytes: number | null;
+    percent: number | null;
+}) => void;
+/**
+ * Checks if semantic data is already downloaded
+ */
+export declare function isSemanticDataDownloaded(): Promise<boolean>;
+/**
+ * Checks if the semantic data files are available
+ * Note: This is now async to work with browser storage
+ */
+export declare function isSemanticDataAvailable(): Promise<boolean>;
+/**
+ * Downloads all semantic data files
+ */
+export declare function downloadSemanticData(onProgress?: SemanticDownloadProgressCallback, onStatus?: (status: string) => void): Promise<string>;
+/**
+ * Ensures semantic data is available, downloading if needed
+ */
+export declare function ensureSemanticData(options?: {
+    autoDownload?: boolean;
+    onProgress?: SemanticDownloadProgressCallback;
+    onStatus?: (status: string) => void;
+}): Promise<string>;
+/**
+ * Clears cached semantic data
+ */
+export declare function clearSemanticDataCache(): Promise<void>;
+/**
+ * Gets info about semantic data files
+ */
+export declare function getSemanticDataInfo(): Promise<{
+    files: SemanticDataFileInfo[];
+    cacheDir: string;
+    totalSize: string;
+}>;
+/**
+ * Initializes semantic data (downloads if needed, then loads)
+ */
+export declare function initializeSemanticData(options?: {
+    autoDownload?: boolean;
+    onProgress?: SemanticDownloadProgressCallback;
+    onStatus?: (status: string) => void;
+}): Promise<void>;
+/**
+ * Loads semantic data from cached files
+ * @throws Error if required data files are not available
+ */
+export declare function loadSemanticData(): Promise<SemanticData>;
+/**
+ * Gets the loaded semantic data (loads if not already loaded)
+ * Note: This is now async to work with browser storage
+ */
+export declare function getSemanticData(): Promise<SemanticData>;
+/**
+ * Gets the loaded semantic data synchronously
+ * Returns null if data is not yet loaded
+ * Use this for performance-critical code paths after initial loading
+ */
+export declare function getSemanticDataSync(): SemanticData | null;
+/**
+ * Clears the loaded semantic data (useful for testing)
+ */
+export declare function clearSemanticData(): void;
+/**
+ * Looks up gender for a name
+ */
+export declare function lookupGender(name: string, locale?: string): Promise<string | undefined>;
+/**
+ * Looks up location type (city, country, or region)
+ * Priority: country > major city (pop > 500K) > region > other cities
+ */
+export declare function lookupLocationType(location: string): Promise<{
+    type: "city" | "country" | "region";
+    countryCode?: string;
+} | undefined>;
+/**
+ * Gets statistics about loaded data
+ */
+export declare function getDataStats(): {
+    names: number;
+    cities: number;
+    countries: number;
+    regions: number;
+    loaded: boolean;
+};
+export {};
+//# sourceMappingURL=semantic-data-loader.d.ts.map

package/dist/pipeline/semantic-data-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-data-loader.d.ts","sourceRoot":"","sources":["../../src/pipeline/semantic-data-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA6EH;;GAEG;AACH,UAAU,SAAS;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C;AAED;;GAEG;AACH,UAAU,SAAS;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,UAAU,YAAY;IACpB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC9B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC/B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAClC,MAAM,EAAE,OAAO,CAAC;CACjB;AAsBD;;;GAGG;AACH,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,MAAM,CAAC,CAG/D;AAED;;GAEG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAExD;AAMD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,eAAe;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,+BAA+B;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,kBAAkB;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;AAQD;;GAEG;AACH,eAAO,MAAM,mBAAmB,EAAE,oBAAoB,EA6BrD,CAAC;AAMF;;GAEG;AACH,MAAM,MAAM,gCAAgC,GAAG,CAAC,QAAQ,EAAE;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,KAAK,IAAI,CAAC;AAEX;;GAEG;AACH,wBAAsB,wBAAwB,IAAI,OAAO,CAAC,OAAO,CAAC,CAgBjE;AAED;;;GAGG;AACH,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,OAAO,CAAC,CAEhE;AAuFD;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,UAAU,CAAC,EAAE,gCAAgC,EAC7C,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,GAClC,OAAO,CAAC,MAAM,CAAC,CA+BjB;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,gCAAgC,CAAC;IAC9C,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC,GACL,OAAO,CAAC,MAAM,CAAC,CAyBjB;AAED;;GAEG;AACH,wBAAsB,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC,CAM5D;AAED;;GAEG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC;IACnD,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC,CAOD;AAqPD;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,gCAAgC,CAAC;IAC9C,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC,GACL,OAAO,CAAC,IAAI,CAAC,CAMf;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,YAAY,CAAC,CAqD9D;AAED;;;GAGG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC,CAK7D;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,IAAI,YAAY,GAAG,IAAI,CAKzD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAiB7B;AAOD;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,GACf,OAAO,CACR;IAAE,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAC1E,CA4BA;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB,CAYA"}