rehydra 0.1.0

package/dist/pipeline/tagger.js

@@ -0,0 +1,374 @@
+/**
+ * Replacement Tagger
+ * Replaces PII spans with placeholder tags and builds the PII map
+ */
+import { PIIType, } from "../types/index.js";
+import { sortSpansByPosition } from "../utils/offsets.js";
+/**
+ * Generates a PII placeholder tag
+ * Format: <PII type="TYPE" id="N"/> or <PII type="TYPE" gender="X" id="N"/> etc.
+ *
+ * Semantic attributes (gender, scope) are included when provided and not 'unknown'
+ */
+export function generateTag(type, id, semantic) {
+    let attrs = `type="${type}"`;
+    // Add semantic attributes if present and meaningful
+    if (semantic?.gender && semantic.gender !== "unknown") {
+        attrs += ` gender="${semantic.gender}"`;
+    }
+    if (semantic?.scope && semantic.scope !== "unknown") {
+        attrs += ` scope="${semantic.scope}"`;
+    }
+    attrs += ` id="${id}"`;
+    return `<PII ${attrs}/>`;
+}
+/**
+ * Parses a PII tag to extract type, id, and semantic attributes
+ * Returns null if not a valid tag
+ *
+ * Supports formats:
+ * - <PII type="TYPE" id="N"/>
+ * - <PII type="TYPE" gender="X" id="N"/>
+ * - <PII type="TYPE" scope="X" id="N"/>
+ * - <PII type="TYPE" gender="X" scope="Y" id="N"/>
+ */
+export function parseTag(tag) {
+    // More flexible regex that handles optional gender/scope attributes
+    const match = tag.match(/^<PII\s+type="([A-Z_]+)"(?:\s+gender="(\w+)")?(?:\s+scope="(\w+)")?\s+id="(\d+)"\s*\/>$/);
+    if (match === null) {
+        return null;
+    }
+    const [, typeStr, genderStr, scopeStr, idStr] = match;
+    if (typeStr === undefined || idStr === undefined) {
+        return null;
+    }
+    const type = typeStr;
+    const id = parseInt(idStr, 10);
+    // Validate type is a valid PIIType
+    if (!Object.values(PIIType).includes(type)) {
+        return null;
+    }
+    // Build semantic attributes if present
+    let semantic;
+    if ((genderStr !== undefined && genderStr !== "") ||
+        (scopeStr !== undefined && scopeStr !== "")) {
+        semantic = {};
+        if (genderStr !== undefined &&
+            genderStr !== "" &&
+            ["male", "female", "neutral", "unknown"].includes(genderStr)) {
+            semantic.gender = genderStr;
+        }
+        if (scopeStr !== undefined &&
+            scopeStr !== "" &&
+            ["city", "country", "region", "unknown"].includes(scopeStr)) {
+            semantic.scope = scopeStr;
+        }
+    }
+    return { type, id, semantic };
+}
+/**
+ * Creates a key for the PII map
+ */
+export function createPIIMapKey(type, id) {
+    return `${type}_${id}`;
+}
+/**
+ * Tags PII spans in text and builds the PII map
+ */
+export function tagEntities(text, matches, policy) {
+    if (matches.length === 0) {
+        return {
+            anonymizedText: text,
+            entities: [],
+            piiMap: new Map(),
+        };
+    }
+    // Sort by start position ascending for ID assignment
+    const sortedAscending = sortSpansByPosition(matches);
+    // Assign IDs
+    const entitiesWithIds = [];
+    let nextId = 1;
+    // Track seen text for ID reuse (if enabled)
+    const seenText = new Map(); // text -> id
+    for (const match of sortedAscending) {
+        let id;
+        if (policy.reuseIdsForRepeatedPII) {
+            const key = `${match.type}:${match.text}`;
+            const existingId = seenText.get(key);
+            if (existingId !== undefined) {
+                id = existingId;
+            }
+            else {
+                id = nextId++;
+                seenText.set(key, id);
+            }
+        }
+        else {
+            id = nextId++;
+        }
+        entitiesWithIds.push({ ...match, id });
+    }
+    // Build PII map
+    const piiMap = new Map();
+    for (const entity of entitiesWithIds) {
+        const key = createPIIMapKey(entity.type, entity.id);
+        piiMap.set(key, entity.text);
+    }
+    // Sort by start position descending for replacement
+    // (replacing from end to start preserves earlier offsets)
+    const sortedDescending = [...entitiesWithIds].sort((a, b) => b.start - a.start);
+    // Perform replacements
+    let anonymizedText = text;
+    for (const entity of sortedDescending) {
+        const tag = generateTag(entity.type, entity.id, entity.semantic);
+        anonymizedText =
+            anonymizedText.slice(0, entity.start) +
+                tag +
+                anonymizedText.slice(entity.end);
+    }
+    // Build final entities list (sorted by position)
+    const entities = entitiesWithIds.map((e) => ({
+        type: e.type,
+        id: e.id,
+        start: e.start,
+        end: e.end,
+        confidence: e.confidence,
+        source: e.source,
+        original: e.text,
+        semantic: e.semantic,
+    }));
+    return {
+        anonymizedText,
+        entities: sortSpansByPosition(entities),
+        piiMap,
+    };
+}
+/**
+ * Validates that a tag is well-formed
+ */
+export function isValidTag(tag) {
+    return parseTag(tag) !== null;
+}
+/**
+ * Quote characters that might appear after translation
+ * Includes: standard quotes, smart quotes, German quotes, French quotes, etc.
+ *
+ * Unicode references:
+ * - \u0022 (") Standard double quote
+ * - \u0027 (') Standard single quote
+ * - \u0060 (`) Backtick
+ * - \u00AB («) Left guillemet
+ * - \u00BB (») Right guillemet
+ * - \u2018 (') Left single curly quote
+ * - \u2019 (') Right single curly quote
+ * - \u201A (‚) Single low-9 quote
+ * - \u201C (") Left double curly quote
+ * - \u201D (") Right double curly quote
+ * - \u201E („) Double low-9 quote (German)
+ */
+const QUOTE_CHARS = "[\"'`\u00AB\u00BB\u2018\u2019\u201A\u201C\u201D\u201E]";
+/**
+ * Whitespace pattern including various unicode spaces
+ */
+const FLEXIBLE_WS = `[\\s\\u00A0\\u2000-\\u200B]*`;
+const FLEXIBLE_WS_REQUIRED = `[\\s\\u00A0\\u2000-\\u200B]+`;
+/**
+ * Builds patterns for fuzzy PII tag matching
+ * Handles various translation artifacts and optional semantic attributes
+ */
+function buildFuzzyTagPatterns() {
+    // Pattern for type attribute: type = "VALUE" (flexible spacing and quotes)
+    const typeAttr = `type${FLEXIBLE_WS}=${FLEXIBLE_WS}${QUOTE_CHARS}([A-Z_]+)${QUOTE_CHARS}`;
+    // Pattern for id attribute: id = "VALUE" (flexible spacing and quotes)
+    const idAttr = `id${FLEXIBLE_WS}=${FLEXIBLE_WS}${QUOTE_CHARS}(\\d+)${QUOTE_CHARS}`;
+    // Optional gender attribute
+    const genderAttr = `(?:${FLEXIBLE_WS}gender${FLEXIBLE_WS}=${FLEXIBLE_WS}${QUOTE_CHARS}(\\w+)${QUOTE_CHARS})?`;
+    // Optional scope attribute
+    const scopeAttr = `(?:${FLEXIBLE_WS}scope${FLEXIBLE_WS}=${FLEXIBLE_WS}${QUOTE_CHARS}(\\w+)${QUOTE_CHARS})?`;
+    // Self-closing tag endings: />, / >, >, etc.
+    const selfClosing = `${FLEXIBLE_WS}\\/?${FLEXIBLE_WS}>`;
+    return [
+        // type first with optional gender/scope: <PII type="X" gender="Y" scope="Z" id="N"/>
+        // Groups: type=1, gender=2, scope=3, id=4
+        new RegExp(`<${FLEXIBLE_WS}PII${FLEXIBLE_WS_REQUIRED}${typeAttr}${genderAttr}${scopeAttr}${FLEXIBLE_WS_REQUIRED}${idAttr}${selfClosing}`, "gi"),
+        // id first: <PII id="N" type="X"/>
+        // Groups: id=1, type=2
+        new RegExp(`<${FLEXIBLE_WS}PII${FLEXIBLE_WS_REQUIRED}${idAttr}${FLEXIBLE_WS_REQUIRED}${typeAttr}${selfClosing}`, "gi"),
+    ];
+}
+/**
+ * Extracts all PII tags from anonymized text using fuzzy matching
+ * Handles mangled tags that may occur after translation
+ *
+ * Translation can mangle tags by:
+ * - Changing quote types (" → " or „ or « etc.)
+ * - Adding/removing whitespace
+ * - Changing case (type → Type, PII → pii)
+ * - Reordering attributes (id before type)
+ * - Modifying self-closing syntax (/> → / > or >)
+ */
+export function extractTags(anonymizedText) {
+    const tags = [];
+    const patterns = buildFuzzyTagPatterns();
+    // Track positions we've already matched to avoid duplicates from overlapping patterns
+    const matchedPositions = new Set();
+    for (let patternIndex = 0; patternIndex < patterns.length; patternIndex++) {
+        const pattern = patterns[patternIndex];
+        if (pattern === undefined)
+            continue;
+        let match;
+        // Reset lastIndex for each pattern
+        pattern.lastIndex = 0;
+        while ((match = pattern.exec(anonymizedText)) !== null) {
+            if (matchedPositions.has(match.index)) {
+                continue; // Skip duplicates from overlapping patterns
+            }
+            // Extract type, id, and semantic attributes based on which pattern matched
+            // Pattern 0: type first with optional gender/scope (groups: type=1, gender=2, scope=3, id=4)
+            // Pattern 1: id first (groups: id=1, type=2)
+            let typeStr;
+            let idStr;
+            let genderStr;
+            let scopeStr;
+            if (patternIndex === 0) {
+                typeStr = match[1];
+                genderStr = match[2];
+                scopeStr = match[3];
+                idStr = match[4];
+            }
+            else {
+                idStr = match[1];
+                typeStr = match[2];
+            }
+            if (typeStr !== undefined && idStr !== undefined) {
+                const type = typeStr.toUpperCase();
+                const id = parseInt(idStr, 10);
+                if (Object.values(PIIType).includes(type)) {
+                    // Build semantic attributes if present
+                    let semantic;
+                    if ((genderStr !== undefined && genderStr !== "") ||
+                        (scopeStr !== undefined && scopeStr !== "")) {
+                        semantic = {};
+                        if (genderStr !== undefined &&
+                            genderStr !== "" &&
+                            ["male", "female", "neutral", "unknown"].includes(genderStr.toLowerCase())) {
+                            semantic.gender =
+                                genderStr.toLowerCase();
+                        }
+                        if (scopeStr !== undefined &&
+                            scopeStr !== "" &&
+                            ["city", "country", "region", "unknown"].includes(scopeStr.toLowerCase())) {
+                            semantic.scope =
+                                scopeStr.toLowerCase();
+                        }
+                    }
+                    tags.push({
+                        type,
+                        id,
+                        position: match.index,
+                        matchedText: match[0],
+                        semantic,
+                    });
+                    matchedPositions.add(match.index);
+                }
+            }
+        }
+    }
+    // Sort by position ascending
+    tags.sort((a, b) => a.position - b.position);
+    return tags;
+}
+/**
+ * Extracts tags using strict matching (original behavior)
+ * Useful when you know tags haven't been mangled
+ * Supports optional gender and scope attributes
+ */
+export function extractTagsStrict(anonymizedText) {
+    const tags = [];
+    // Pattern matches: <PII type="X" [gender="Y"] [scope="Z"] id="N"/>
+    const tagPattern = /<PII\s+type="([A-Z_]+)"(?:\s+gender="(\w+)")?(?:\s+scope="(\w+)")?\s+id="(\d+)"\s*\/>/g;
+    let match;
+    while ((match = tagPattern.exec(anonymizedText)) !== null) {
+        const typeStr = match[1];
+        const genderStr = match[2];
+        const scopeStr = match[3];
+        const idStr = match[4];
+        if (typeStr !== undefined && idStr !== undefined) {
+            const type = typeStr;
+            const id = parseInt(idStr, 10);
+            if (Object.values(PIIType).includes(type)) {
+                // Build semantic attributes if present
+                let semantic;
+                if ((genderStr !== undefined && genderStr !== "") ||
+                    (scopeStr !== undefined && scopeStr !== "")) {
+                    semantic = {};
+                    if (genderStr !== undefined &&
+                        genderStr !== "" &&
+                        ["male", "female", "neutral", "unknown"].includes(genderStr)) {
+                        semantic.gender = genderStr;
+                    }
+                    if (scopeStr !== undefined &&
+                        scopeStr !== "" &&
+                        ["city", "country", "region", "unknown"].includes(scopeStr)) {
+                        semantic.scope = scopeStr;
+                    }
+                }
+                tags.push({
+                    type,
+                    id,
+                    position: match.index,
+                    matchedText: match[0],
+                    semantic,
+                });
+            }
+        }
+    }
+    return tags;
+}
+/**
+ * Counts entities by type
+ */
+export function countEntitiesByType(entities) {
+    const counts = {};
+    // Initialize all types to 0
+    for (const type of Object.values(PIIType)) {
+        counts[type] = 0;
+    }
+    // Count entities
+    for (const entity of entities) {
+        counts[entity.type] = (counts[entity.type] ?? 0) + 1;
+    }
+    return counts;
+}
+/**
+ * Rehydrates anonymized text using the PII map
+ * Uses fuzzy matching to handle tags that may have been mangled by translation
+ *
+ * @param anonymizedText - Text containing PII tags (possibly mangled)
+ * @param piiMap - Map of PII keys to original values
+ * @param strict - If true, use strict matching (original behavior). Default: false
+ * @returns Text with PII tags replaced by original values
+ */
+export function rehydrate(anonymizedText, piiMap, strict = false) {
+    let result = anonymizedText;
+    const tags = strict
+        ? extractTagsStrict(anonymizedText)
+        : extractTags(anonymizedText);
+    // Sort by position descending for replacement
+    // (replacing from end to start preserves earlier offsets)
+    tags.sort((a, b) => b.position - a.position);
+    for (const { type, id, position, matchedText } of tags) {
+        const key = createPIIMapKey(type, id);
+        const original = piiMap.get(key);
+        if (original !== undefined) {
+            // Use the actual matched text length for replacement
+            // This handles mangled tags where the length differs from the canonical form
+            result =
+                result.slice(0, position) +
+                    original +
+                    result.slice(position + matchedText.length);
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=tagger.js.map

package/dist/pipeline/tagger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tagger.js","sourceRoot":"","sources":["../../src/pipeline/tagger.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,OAAO,GAKR,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AA+B1D;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CACzB,IAAa,EACb,EAAU,EACV,QAA6B;IAE7B,IAAI,KAAK,GAAG,SAAS,IAAI,GAAG,CAAC;IAE7B,oDAAoD;IACpD,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACtD,KAAK,IAAI,YAAY,QAAQ,CAAC,MAAM,GAAG,CAAC;IAC1C,CAAC;IACD,IAAI,QAAQ,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACpD,KAAK,IAAI,WAAW,QAAQ,CAAC,KAAK,GAAG,CAAC;IACxC,CAAC;IAED,KAAK,IAAI,QAAQ,EAAE,GAAG,CAAC;IAEvB,OAAO,QAAQ,KAAK,IAAI,CAAC;AAC3B,CAAC;AAWD;;;;;;;;;GASG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,oEAAoE;IACpE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CACrB,yFAAyF,CAC1F,CAAC;IAEF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,GAAG,KAAK,CAAC;IACtD,IAAI,OAAO,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,OAAkB,CAAC;IAChC,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAE/B,mCAAmC;IACnC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uCAAuC;IACvC,IAAI,QAAwC,CAAC;IAC7C,IACE,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE,CAAC;QAC7C,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,EAAE,CAAC,EAC3C,CAAC;QACD,QAAQ,GAAG,EAAE,CAAC;QACd,IACE,SAAS,KAAK,SAAS;YACvB,SAAS,KAAK,EAAE;YAChB,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAC5D,CAAC;YACD,QAAQ,CAAC,MAAM,GAAG,SAAyC,CAAC;QAC9D,CAAC;QACD,IACE,QAAQ,KAAK,SAAS;YACtB,QAAQ,KAAK,EAAE;YACf,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC3D,CAAC;YACD,QAAQ,CAAC,KAAK,GAAG,QAAuC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,IAAa,EAAE,EAAU;IACvD,OAAO,GAAG,IAAI,IAAI,EAAE,EAAE,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CACzB,IAAY,EACZ,OAAoB,EACpB,MAA2B;IAE3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,IAAI,GAAG,EAAE;SAClB,CAAC;IACJ,CAAC;IAED,qDAAqD;IACrD,MAAM,eAAe,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAErD,aAAa;IACb,MAAM,eAAe,GAAsC,EAAE,CAAC;IAC9D,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,4CAA4C;IAC5C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,aAAa;IAEzD,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;QACpC,IAAI,EAAU,CAAC;QAEf,IAAI,MAAM,CAAC,sBAAsB,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC1C,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,EAAE,GAAG,UAAU,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,EAAE,GAAG,MAAM,EAAE,CAAC;gBACd,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,MAAM,EAAE,CAAC;QAChB,CAAC;QAED,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,gBAAgB;IAChB,MAAM,MAAM,GAAc,IAAI,GAAG,EAAE,CAAC;IACpC,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACpD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,oDAAoD;IACpD,0DAA0D;IAC1D,MAAM,gBAAgB,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC,IAAI,CAChD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAC5B,CAAC;IAEF,uBAAuB;IACvB,IAAI,cAAc,GAAG,IAAI,CAAC;IAC1B,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QACjE,cAAc;YACZ,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;gBACrC,GAAG;gBACH,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,iDAAiD;IACjD,MAAM,QAAQ,GAAqB,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7D,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,IAAI;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACrB,CAAC,CAAC,CAAC;IAEJ,OAAO;QACL,cAAc;QACd,QAAQ,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QACvC,MAAM;KACP,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,OAAO,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC;AAChC,CAAC;AAeD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,GAAG,wDAAwD,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,GAAG,8BAA8B,CAAC;AACnD,MAAM,oBAAoB,GAAG,8BAA8B,CAAC;AAE5D;;;GAGG;AACH,SAAS,qBAAqB;IAC5B,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,OAAO,WAAW,IAAI,WAAW,GAAG,WAAW,YAAY,WAAW,EAAE,CAAC;IAC1F,uEAAuE;IACvE,MAAM,MAAM,GAAG,KAAK,WAAW,IAAI,WAAW,GAAG,WAAW,SAAS,WAAW,EAAE,CAAC;IACnF,4BAA4B;IAC5B,MAAM,UAAU,GAAG,MAAM,WAAW,SAAS,WAAW,IAAI,WAAW,GAAG,WAAW,SAAS,WAAW,IAAI,CAAC;IAC9G,2BAA2B;IAC3B,MAAM,SAAS,GAAG,MAAM,WAAW,QAAQ,WAAW,IAAI,WAAW,GAAG,WAAW,SAAS,WAAW,IAAI,CAAC;IAE5G,6CAA6C;IAC7C,MAAM,WAAW,GAAG,GAAG,WAAW,OAAO,WAAW,GAAG,CAAC;IAExD,OAAO;QACL,qFAAqF;QACrF,0CAA0C;QAC1C,IAAI,MAAM,CACR,IAAI,WAAW,MAAM,oBAAoB,GAAG,QAAQ,GAAG,UAAU,GAAG,SAAS,GAAG,oBAAoB,GAAG,MAAM,GAAG,WAAW,EAAE,EAC7H,IAAI,CACL;QACD,mCAAmC;QACnC,uBAAuB;QACvB,IAAI,MAAM,CACR,IAAI,WAAW,MAAM,oBAAoB,GAAG,MAAM,GAAG,oBAAoB,GAAG,QAAQ,GAAG,WAAW,EAAE,EACpG,IAAI,CACL;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,WAAW,CAAC,cAAsB;IAChD,MAAM,IAAI,GAAmB,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAG,qBAAqB,EAAE,CAAC;IAEzC,sFAAsF;IACtF,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE3C,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;QACvC,IAAI,OAAO,KAAK,SAAS;YAAE,SAAS;QAEpC,IAAI,KAA6B,CAAC;QAClC,mCAAmC;QACnC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QAEtB,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACvD,IAAI,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtC,SAAS,CAAC,4CAA4C;YACxD,CAAC;YAED,2EAA2E;YAC3E,6FAA6F;YAC7F,6CAA6C;YAC7C,IAAI,OAA2B,CAAC;YAChC,IAAI,KAAyB,CAAC;YAC9B,IAAI,SAA6B,CAAC;YAClC,IAAI,QAA4B,CAAC;YAEjC,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACnB,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACpB,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjB,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACrB,CAAC;YAED,IAAI,OAAO,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACjD,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAa,CAAC;gBAC9C,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC1C,uCAAuC;oBACvC,IAAI,QAAwC,CAAC;oBAC7C,IACE,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE,CAAC;wBAC7C,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,EAAE,CAAC,EAC3C,CAAC;wBACD,QAAQ,GAAG,EAAE,CAAC;wBACd,IACE,SAAS,KAAK,SAAS;4BACvB,SAAS,KAAK,EAAE;4BAChB,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAC/C,SAAS,CAAC,WAAW,EAAE,CACxB,EACD,CAAC;4BACD,QAAQ,CAAC,MAAM;gCACb,SAAS,CAAC,WAAW,EAAkC,CAAC;wBAC5D,CAAC;wBACD,IACE,QAAQ,KAAK,SAAS;4BACtB,QAAQ,KAAK,EAAE;4BACf,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,QAAQ,CAC/C,QAAQ,CAAC,WAAW,EAAE,CACvB,EACD,CAAC;4BACD,QAAQ,CAAC,KAAK;gCACZ,QAAQ,CAAC,WAAW,EAAiC,CAAC;wBAC1D,CAAC;oBACH,CAAC;oBAED,IAAI,CAAC,IAAI,CAAC;wBACR,IAAI;wBACJ,EAAE;wBACF,QAAQ,EAAE,KAAK,CAAC,KAAK;wBACrB,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;wBACrB,QAAQ;qBACT,CAAC,CAAC;oBACH,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACpC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,cAAsB;IACtD,MAAM,IAAI,GAAmB,EAAE,CAAC;IAChC,mEAAmE;IACnE,MAAM,UAAU,GACd,wFAAwF,CAAC;IAE3F,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEvB,IAAI,OAAO,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACjD,MAAM,IAAI,GAAG,OAAkB,CAAC;YAChC,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAE/B,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1C,uCAAuC;gBACvC,IAAI,QAAwC,CAAC;gBAC7C,IACE,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE,CAAC;oBAC7C,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,EAAE,CAAC,EAC3C,CAAC;oBACD,QAAQ,GAAG,EAAE,CAAC;oBACd,IACE,SAAS,KAAK,SAAS;wBACvB,SAAS,KAAK,EAAE;wBAChB,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAC5D,CAAC;wBACD,QAAQ,CAAC,MAAM,GAAG,SAAyC,CAAC;oBAC9D,CAAC;oBACD,IACE,QAAQ,KAAK,SAAS;wBACtB,QAAQ,KAAK,EAAE;wBACf,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC3D,CAAC;wBACD,QAAQ,CAAC,KAAK,GAAG,QAAuC,CAAC;oBAC3D,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC;oBACR,IAAI;oBACJ,EAAE;oBACF,QAAQ,EAAE,KAAK,CAAC,KAAK;oBACrB,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;oBACrB,QAAQ;iBACT,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAA0B;IAE1B,MAAM,MAAM,GAA4B,EAA6B,CAAC;IAEtE,4BAA4B;IAC5B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAED,iBAAiB;IACjB,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CACvB,cAAsB,EACtB,MAAiB,EACjB,SAAkB,KAAK;IAEvB,IAAI,MAAM,GAAG,cAAc,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM;QACjB,CAAC,CAAC,iBAAiB,CAAC,cAAc,CAAC;QACnC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;IAEhC,8CAA8C;IAC9C,0DAA0D;IAC1D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAE7C,KAAK,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC;QACvD,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,qDAAqD;YACrD,6EAA6E;YAC7E,MAAM;gBACJ,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC;oBACzB,QAAQ;oBACR,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/pipeline/title-extractor.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Title Extractor
+ * Extracts and strips honorific titles/prefixes from PERSON entities
+ * so that titles remain visible in anonymized text for translation.
+ *
+ * Supported languages: ar, de, en, es, fr, it, lv, nl, pt, zh
+ */
+import { SpanMatch, SemanticAttributes } from "../types/index.js";
+/**
+ * Title extraction result
+ */
+export interface TitleExtractionResult {
+    /** The extracted title (e.g., "Dr.", "Mr.") or undefined if no title */
+    title?: string;
+    /** The name without the title */
+    nameWithoutTitle: string;
+    /** Character offset where the name starts (after title + space) */
+    titleLength: number;
+}
+/**
+ * Extracts a title from the beginning of a name
+ *
+ * @param name - Full name potentially starting with a title
+ * @returns Extraction result with title, remaining name, and offset
+ *
+ * @example
+ * extractTitle("Dr. John Smith") // { title: "Dr.", nameWithoutTitle: "John Smith", titleLength: 4 }
+ * extractTitle("John Smith") // { title: undefined, nameWithoutTitle: "John Smith", titleLength: 0 }
+ */
+export declare function extractTitle(name: string): TitleExtractionResult;
+/**
+ * Extended semantic attributes including title
+ */
+export interface SemanticAttributesWithTitle extends SemanticAttributes {
+    /** Extracted title prefix (e.g., "Dr.", "Mrs.") */
+    title?: string;
+}
+/**
+ * Processes PERSON spans to extract titles
+ * Titles are removed from the span and stored in semantic attributes
+ * The span boundaries are adjusted so the title remains visible
+ *
+ * @param spans - Array of detected PII spans
+ * @param originalText - The original text (needed to verify span boundaries)
+ * @returns Array of spans with titles extracted from PERSON entities
+ */
+export declare function extractTitlesFromSpans(spans: SpanMatch[], originalText: string): SpanMatch[];
+/**
+ * Gets all supported titles for a specific language
+ */
+export declare function getTitlesForLanguage(langCode: "ar" | "de" | "en" | "es" | "fr" | "it" | "lv" | "nl" | "pt" | "zh"): string[];
+/**
+ * Gets all supported titles across all languages
+ */
+export declare function getAllTitles(): string[];
+/**
+ * Checks if a string starts with a known title
+ */
+export declare function startsWithTitle(text: string): boolean;
+/**
+ * Checks if a text consists entirely of a title (with optional punctuation)
+ */
+export declare function isOnlyTitle(text: string): boolean;
+/**
+ * Merges adjacent PERSON spans when one is a title
+ *
+ * This fixes issues where NER models split "Mrs. Smith" into two entities:
+ * - PERSON: "Mrs" (or "Mrs.")
+ * - PERSON: "Smith"
+ *
+ * After merging: PERSON: "Mrs. Smith"
+ *
+ * @param spans - Array of detected PII spans
+ * @param originalText - The original text
+ * @param maxGap - Maximum characters between spans to consider them adjacent (default: 3)
+ * @returns Array of spans with adjacent title+name PERSON entities merged
+ */
+export declare function mergeAdjacentTitleSpans(spans: SpanMatch[], originalText: string, maxGap?: number): SpanMatch[];
+//# sourceMappingURL=title-extractor.d.ts.map

package/dist/pipeline/title-extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"title-extractor.d.ts","sourceRoot":"","sources":["../../src/pipeline/title-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,SAAS,EAET,kBAAkB,EAEnB,MAAM,mBAAmB,CAAC;AAE3B;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,wEAAwE;IACxE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,gBAAgB,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,WAAW,EAAE,MAAM,CAAC;CACrB;AAqjBD;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,CAyBhE;AAED;;GAEG;AACH,MAAM,WAAW,2BAA4B,SAAQ,kBAAkB;IACrE,mDAAmD;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,SAAS,EAAE,EAClB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CAiEb;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAC5E,MAAM,EAAE,CAcV;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,MAAM,EAAE,CAEvC;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAGrD;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CA0BjD;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,EAAE,EAClB,YAAY,EAAE,MAAM,EACpB,MAAM,GAAE,MAAU,GACjB,SAAS,EAAE,CAuFb"}