rehydra 0.1.0

package/dist/pipeline/semantic-enricher.d.ts

@@ -0,0 +1,112 @@
+/**
+ * Semantic Enricher
+ * Enriches PII spans with semantic attributes (gender, location scope)
+ * for MT-friendly tags that preserve grammatical context.
+ *
+ * This module uses data from the GeoNames and gender-guesser projects.
+ * Data is automatically downloaded when using:
+ *   createAnonymizer({ semantic: { enabled: true, autoDownload: true } })
+ */
+import { SpanMatch, PersonGender, LocationScope } from "../types/index.js";
+/**
+ * Configuration for semantic enrichment
+ */
+export interface EnricherConfig {
+    /** Locale hint for name gender disambiguation (e.g., 'de', 'it', 'fr') */
+    locale?: string;
+    /** Minimum confidence to apply semantic attributes (default: 0.0) */
+    minConfidence?: number;
+    /** Whether to mark low-confidence results as 'unknown' */
+    strictMode?: boolean;
+}
+/**
+ * Result of gender inference with confidence
+ */
+export interface GenderResult {
+    gender: PersonGender;
+    confidence: number;
+    source: "database" | "inference" | "unknown";
+}
+/**
+ * Result of location classification with confidence
+ */
+export interface LocationResult {
+    scope: LocationScope;
+    confidence: number;
+    countryCode?: string;
+}
+/**
+ * Initializes semantic data (async, must be called before sync functions)
+ * @throws Error if data files are not available
+ */
+export declare function initializeEnricher(): Promise<void>;
+/**
+ * Checks if enricher is ready for synchronous operations
+ */
+export declare function isEnricherReady(): boolean;
+/**
+ * Enriches PII spans with semantic attributes based on lookup tables
+ *
+ * NOTE: This function requires semantic data to be pre-loaded via initializeEnricher()
+ * or through createAnonymizer({ semantic: { enabled: true } }).
+ *
+ * @param spans - Array of detected PII spans
+ * @param config - Optional configuration for enrichment
+ * @returns Array of spans with semantic attributes added
+ *
+ * @example
+ * ```typescript
+ * const enrichedSpans = enrichSemantics(spans, { locale: 'de' });
+ * // "Mary" -> { gender: 'female' }
+ * // "Berlin" -> { scope: 'city' }
+ * ```
+ */
+export declare function enrichSemantics(spans: SpanMatch[], config?: EnricherConfig): SpanMatch[];
+/**
+ * Infers gender from a person's name using the lookup database
+ *
+ * @param name - Full name or first name
+ * @param locale - Optional locale for disambiguation (e.g., 'de', 'it')
+ * @returns Gender result with confidence
+ *
+ * @example
+ * ```typescript
+ * inferGender('Mary Smith'); // { gender: 'female', confidence: 1.0 }
+ * inferGender('Andrea', 'it'); // { gender: 'male', confidence: 1.0 }
+ * inferGender('Andrea', 'en'); // { gender: 'female', confidence: 1.0 }
+ * ```
+ */
+export declare function inferGender(name: string, locale?: string): GenderResult;
+/**
+ * Classifies a location by its geographic scope
+ *
+ * @param location - Location name
+ * @returns Classification result with confidence
+ *
+ * @example
+ * ```typescript
+ * classifyLocation('Berlin'); // { scope: 'city', confidence: 1.0 }
+ * classifyLocation('Germany'); // { scope: 'country', confidence: 1.0 }
+ * classifyLocation('Bavaria'); // { scope: 'region', confidence: 1.0 }
+ * ```
+ */
+export declare function classifyLocation(location: string): LocationResult;
+/**
+ * Gets statistics about the lookup databases
+ */
+export declare function getDatabaseStats(): {
+    names: number;
+    cities: number;
+    countries: number;
+    regions: number;
+    loaded: boolean;
+};
+/**
+ * Checks if a name exists in the database
+ */
+export declare function hasName(name: string): boolean;
+/**
+ * Checks if a location exists in the database
+ */
+export declare function hasLocation(location: string): boolean;
+//# sourceMappingURL=semantic-enricher.d.ts.map

package/dist/pipeline/semantic-enricher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-enricher.d.ts","sourceRoot":"","sources":["../../src/pipeline/semantic-enricher.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,SAAS,EAET,YAAY,EACZ,aAAa,EACd,MAAM,mBAAmB,CAAC;AAS3B;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,0EAA0E;IAC1E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0DAA0D;IAC1D,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,GAAG,WAAW,GAAG,SAAS,CAAC;CAC9C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,aAAa,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAKD;;;GAGG;AACH,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAaxD;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,OAAO,CAEzC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,SAAS,EAAE,EAClB,MAAM,CAAC,EAAE,cAAc,GACtB,SAAS,EAAE,CAkBb;AAoGD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,YAAY,CAwBvE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,CAgCjE;AA+ED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB,CAEA;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAU7C;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAQrD"}

package/dist/pipeline/semantic-enricher.js

@@ -0,0 +1,318 @@
+/**
+ * Semantic Enricher
+ * Enriches PII spans with semantic attributes (gender, location scope)
+ * for MT-friendly tags that preserve grammatical context.
+ *
+ * This module uses data from the GeoNames and gender-guesser projects.
+ * Data is automatically downloaded when using:
+ *   createAnonymizer({ semantic: { enabled: true, autoDownload: true } })
+ */
+import { PIIType, } from "../types/index.js";
+import { isSemanticDataDownloaded, loadSemanticData, getSemanticDataSync, getDataStats, } from "./semantic-data-loader.js";
+// Track if data has been initialized
+let dataInitialized = false;
+/**
+ * Initializes semantic data (async, must be called before sync functions)
+ * @throws Error if data files are not available
+ */
+export async function initializeEnricher() {
+    if (dataInitialized)
+        return;
+    const available = await isSemanticDataDownloaded();
+    if (!available) {
+        throw new Error(`Semantic enrichment data not available. ` +
+            `Use ensureSemanticData() or createAnonymizer({ semantic: { enabled: true } }) to download.`);
+    }
+    await loadSemanticData();
+    dataInitialized = true;
+}
+/**
+ * Checks if enricher is ready for synchronous operations
+ */
+export function isEnricherReady() {
+    return dataInitialized && getSemanticDataSync() !== null;
+}
+/**
+ * Enriches PII spans with semantic attributes based on lookup tables
+ *
+ * NOTE: This function requires semantic data to be pre-loaded via initializeEnricher()
+ * or through createAnonymizer({ semantic: { enabled: true } }).
+ *
+ * @param spans - Array of detected PII spans
+ * @param config - Optional configuration for enrichment
+ * @returns Array of spans with semantic attributes added
+ *
+ * @example
+ * ```typescript
+ * const enrichedSpans = enrichSemantics(spans, { locale: 'de' });
+ * // "Mary" -> { gender: 'female' }
+ * // "Berlin" -> { scope: 'city' }
+ * ```
+ */
+export function enrichSemantics(spans, config) {
+    // Check if data is loaded
+    const data = getSemanticDataSync();
+    if (data === null) {
+        // Silently skip enrichment if data not available
+        return spans;
+    }
+    return spans.map((span) => {
+        switch (span.type) {
+            case PIIType.PERSON:
+                return enrichPerson(span, config?.locale);
+            case PIIType.LOCATION:
+                return enrichLocation(span);
+            default:
+                return span;
+        }
+    });
+}
+/**
+ * Enriches a PERSON span with gender attribute
+ */
+function enrichPerson(span, locale) {
+    const result = inferGender(span.text, locale);
+    return {
+        ...span,
+        semantic: {
+            ...span.semantic,
+            gender: result.gender,
+        },
+    };
+}
+/**
+ * Enriches a LOCATION span with scope attribute
+ */
+function enrichLocation(span) {
+    const result = classifyLocation(span.text);
+    return {
+        ...span,
+        semantic: {
+            ...span.semantic,
+            scope: result.scope,
+        },
+    };
+}
+/**
+ * Looks up gender for a name in the database (synchronous)
+ */
+function lookupGenderSync(name, locale) {
+    const data = getSemanticDataSync();
+    if (data === null)
+        return undefined;
+    const entry = data.names.get(name.toLowerCase());
+    if (entry === undefined)
+        return undefined;
+    // Check for locale-specific override
+    if (locale !== undefined &&
+        locale !== "" &&
+        entry.localeOverrides !== undefined &&
+        entry.localeOverrides[locale] !== undefined) {
+        return entry.localeOverrides[locale];
+    }
+    return entry.gender;
+}
+/**
+ * Population threshold for "major" cities that take precedence over regions
+ */
+const MAJOR_CITY_POPULATION = 500000;
+/**
+ * Looks up location type synchronously
+ */
+function lookupLocationTypeSync(location) {
+    const data = getSemanticDataSync();
+    if (data === null)
+        return undefined;
+    const normalized = location.toLowerCase().trim();
+    // Check countries FIRST (to avoid "USA" being matched as a city)
+    const countryCode = data.countries.get(normalized);
+    if (countryCode !== undefined) {
+        return { type: "country", countryCode };
+    }
+    // Check cities - if it's a major city (pop > 500K), prioritize it over regions
+    const city = data.cities.get(normalized);
+    if (city && city.population >= MAJOR_CITY_POPULATION) {
+        return { type: "city", countryCode: city.country };
+    }
+    // Check regions
+    const region = data.regions.get(normalized);
+    if (region) {
+        return { type: "region", countryCode: region.country };
+    }
+    // Check remaining cities (smaller cities)
+    if (city) {
+        return { type: "city", countryCode: city.country };
+    }
+    return undefined;
+}
+/**
+ * Infers gender from a person's name using the lookup database
+ *
+ * @param name - Full name or first name
+ * @param locale - Optional locale for disambiguation (e.g., 'de', 'it')
+ * @returns Gender result with confidence
+ *
+ * @example
+ * ```typescript
+ * inferGender('Mary Smith'); // { gender: 'female', confidence: 1.0 }
+ * inferGender('Andrea', 'it'); // { gender: 'male', confidence: 1.0 }
+ * inferGender('Andrea', 'en'); // { gender: 'female', confidence: 1.0 }
+ * ```
+ */
+export function inferGender(name, locale) {
+    // Extract first name (handles "John Smith" -> "John")
+    const firstName = extractFirstName(name);
+    if (firstName === null || firstName === "") {
+        return { gender: "unknown", confidence: 0, source: "unknown" };
+    }
+    // Check if data is available
+    const data = getSemanticDataSync();
+    if (data === null) {
+        return { gender: "unknown", confidence: 0, source: "unknown" };
+    }
+    const gender = lookupGenderSync(firstName, locale);
+    if (gender === undefined || gender === "") {
+        return { gender: "unknown", confidence: 0, source: "unknown" };
+    }
+    return {
+        gender: gender,
+        confidence: 1.0,
+        source: "database",
+    };
+}
+/**
+ * Classifies a location by its geographic scope
+ *
+ * @param location - Location name
+ * @returns Classification result with confidence
+ *
+ * @example
+ * ```typescript
+ * classifyLocation('Berlin'); // { scope: 'city', confidence: 1.0 }
+ * classifyLocation('Germany'); // { scope: 'country', confidence: 1.0 }
+ * classifyLocation('Bavaria'); // { scope: 'region', confidence: 1.0 }
+ * ```
+ */
+export function classifyLocation(location) {
+    // Check if data is available
+    const data = getSemanticDataSync();
+    if (data === null) {
+        return { scope: "unknown", confidence: 0 };
+    }
+    const normalized = normalizeLocationName(location);
+    const result = lookupLocationTypeSync(normalized);
+    if (!result) {
+        // Try variations
+        const variations = generateLocationVariations(location);
+        for (const variant of variations) {
+            const variantResult = lookupLocationTypeSync(variant);
+            if (variantResult) {
+                return {
+                    scope: variantResult.type,
+                    confidence: 0.9,
+                    countryCode: variantResult.countryCode,
+                };
+            }
+        }
+        return { scope: "unknown", confidence: 0 };
+    }
+    return {
+        scope: result.type,
+        confidence: 1.0,
+        countryCode: result.countryCode,
+    };
+}
+/**
+ * Extracts the first name from a full name
+ */
+function extractFirstName(fullName) {
+    const trimmed = fullName.trim();
+    if (!trimmed)
+        return null;
+    // Handle common prefixes (Dr., Mr., Mrs., etc.)
+    const withoutPrefix = trimmed.replace(/^(dr\.?|mr\.?|mrs\.?|ms\.?|prof\.?|rev\.?|sir|dame|lord|lady)\s+/i, "");
+    // Split and get first word
+    const parts = withoutPrefix.split(/\s+/);
+    return parts[0] ?? null;
+}
+/**
+ * Normalizes a location name for lookup
+ */
+function normalizeLocationName(location) {
+    return (location
+        .toLowerCase()
+        .trim()
+        // Remove common suffixes
+        .replace(/\s+(city|town|village|state|province|region|county)$/i, "")
+        // Normalize whitespace
+        .replace(/\s+/g, " "));
+}
+/**
+ * Generates variations of a location name for fuzzy matching
+ */
+function generateLocationVariations(location) {
+    const normalized = normalizeLocationName(location);
+    const variations = [];
+    // Try without "the"
+    if (normalized.startsWith("the ")) {
+        variations.push(normalized.slice(4));
+    }
+    // Try without common articles in other languages
+    const articlePatterns = [
+        /^(la|le|les|el|los|las|il|lo|gli|i|die|der|das|de|het)\s+/i,
+    ];
+    for (const pattern of articlePatterns) {
+        const withoutArticle = normalized.replace(pattern, "");
+        if (withoutArticle !== normalized) {
+            variations.push(withoutArticle);
+        }
+    }
+    // Try ASCII transliteration for common diacritics
+    const asciiVersion = normalized
+        .replace(/[àáâãäå]/g, "a")
+        .replace(/[èéêë]/g, "e")
+        .replace(/[ìíîï]/g, "i")
+        .replace(/[òóôõö]/g, "o")
+        .replace(/[ùúûü]/g, "u")
+        .replace(/[ñ]/g, "n")
+        .replace(/[ç]/g, "c")
+        .replace(/[ß]/g, "ss")
+        .replace(/[æ]/g, "ae")
+        .replace(/[ø]/g, "o")
+        .replace(/[œ]/g, "oe");
+    if (asciiVersion !== normalized) {
+        variations.push(asciiVersion);
+    }
+    return variations;
+}
+/**
+ * Gets statistics about the lookup databases
+ */
+export function getDatabaseStats() {
+    return getDataStats();
+}
+/**
+ * Checks if a name exists in the database
+ */
+export function hasName(name) {
+    const data = getSemanticDataSync();
+    if (data === null) {
+        return false;
+    }
+    const firstName = extractFirstName(name);
+    if (firstName === null || firstName === "")
+        return false;
+    return lookupGenderSync(firstName) !== undefined;
+}
+/**
+ * Checks if a location exists in the database
+ */
+export function hasLocation(location) {
+    const data = getSemanticDataSync();
+    if (data === null) {
+        return false;
+    }
+    const normalized = normalizeLocationName(location);
+    return lookupLocationTypeSync(normalized) !== undefined;
+}
+//# sourceMappingURL=semantic-enricher.js.map

package/dist/pipeline/semantic-enricher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-enricher.js","sourceRoot":"","sources":["../../src/pipeline/semantic-enricher.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAEL,OAAO,GAGR,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,wBAAwB,EACxB,gBAAgB,EAChB,mBAAmB,EACnB,YAAY,GACb,MAAM,2BAA2B,CAAC;AAgCnC,qCAAqC;AACrC,IAAI,eAAe,GAAG,KAAK,CAAC;AAE5B;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,IAAI,eAAe;QAAE,OAAO;IAE5B,MAAM,SAAS,GAAG,MAAM,wBAAwB,EAAE,CAAC;IACnD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,0CAA0C;YACxC,4FAA4F,CAC/F,CAAC;IACJ,CAAC;IAED,MAAM,gBAAgB,EAAE,CAAC;IACzB,eAAe,GAAG,IAAI,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,OAAO,eAAe,IAAI,mBAAmB,EAAE,KAAK,IAAI,CAAC;AAC3D,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,eAAe,CAC7B,KAAkB,EAClB,MAAuB;IAEvB,0BAA0B;IAC1B,MAAM,IAAI,GAAG,mBAAmB,EAAE,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,iDAAiD;QACjD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAClB,KAAK,OAAO,CAAC,MAAM;gBACjB,OAAO,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YAC5C,KAAK,OAAO,CAAC,QAAQ;gBACnB,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;YAC9B;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,IAAe,EAAE,MAAe;IACpD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE9C,OAAO;QACL,GAAG,IAAI;QACP,QAAQ,EAAE;YACR,GAAG,IAAI,CAAC,QAAQ;YAChB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAe;IACrC,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE3C,OAAO;QACL,GAAG,IAAI;QACP,QAAQ,EAAE;YACR,GAAG,IAAI,CAAC,QAAQ;YAChB,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CACvB,IAAY,EACZ,MAAe;IAEf,MAAM,IAAI,GAAG,mBAAmB,EAAE,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAEpC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAE1C,qCAAqC;IACrC,IACE,MAAM,KAAK,SAAS;QACpB,MAAM,KAAK,EAAE;QACb,KAAK,CAAC,eAAe,KAAK,SAAS;QACnC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,SAAS,EAC3C,CAAC;QACD,OAAO,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAErC;;GAEG;AACH,SAAS,sBAAsB,CAC7B,QAAgB;IAEhB,MAAM,IAAI,GAAG,mBAAmB,EAAE,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAEpC,MAAM,UAAU,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IAEjD,iEAAiE;IACjE,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACnD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;IAC1C,CAAC;IAED,+EAA+E;IAC/E,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACzC,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,qBAAqB,EAAE,CAAC;QACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;IACrD,CAAC;IAED,gBAAgB;IAChB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5C,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IACzD,CAAC;IAED,0CAA0C;IAC1C,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;IACrD,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,MAAe;IACvD,sDAAsD;IACtD,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;QAC3C,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACjE,CAAC;IAED,6BAA6B;IAC7B,MAAM,IAAI,GAAG,mBAAmB,EAAE,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACjE,CAAC;IAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAEnD,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAC1C,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACjE,CAAC;IAED,OAAO;QACL,MAAM,EAAE,MAAsB;QAC9B,UAAU,EAAE,GAAG;QACf,MAAM,EAAE,UAAU;KACnB,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,mBAAmB,EAAE,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IAC7C,CAAC;IAED,MAAM,UAAU,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAElD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,iBAAiB;QACjB,MAAM,UAAU,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;QACxD,KAAK,MAAM,OAAO,IAAI,UAAU,EAAE,CAAC;YACjC,MAAM,aAAa,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;YACtD,IAAI,aAAa,EAAE,CAAC;gBAClB,OAAO;oBACL,KAAK,EAAE,aAAa,CAAC,IAAqB;oBAC1C,UAAU,EAAE,GAAG;oBACf,WAAW,EAAE,aAAa,CAAC,WAAW;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IAC7C,CAAC;IAED,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,IAAqB;QACnC,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,QAAgB;IACxC,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,gDAAgD;IAChD,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,CACnC,mEAAmE,EACnE,EAAE,CACH,CAAC;IAEF,2BAA2B;IAC3B,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,OAAO,CACL,QAAQ;SACL,WAAW,EAAE;SACb,IAAI,EAAE;QACP,yBAAyB;SACxB,OAAO,CAAC,uDAAuD,EAAE,EAAE,CAAC;QACrE,uBAAuB;SACtB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CACxB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,QAAgB;IAClD,MAAM,UAAU,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,oBAAoB;IACpB,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAClC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,iDAAiD;IACjD,MAAM,eAAe,GAAG;QACtB,4DAA4D;KAC7D,CAAC;IACF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;QACtC,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACvD,IAAI,cAAc,KAAK,UAAU,EAAE,CAAC;YAClC,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,MAAM,YAAY,GAAG,UAAU;SAC5B,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC;SACrB,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC;SACrB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAEzB,IAAI,YAAY,KAAK,UAAU,EAAE,CAAC;QAChC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAO9B,OAAO,YAAY,EAAE,CAAC;AACxB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,MAAM,IAAI,GAAG,mBAAmB,EAAE,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC;IAEzD,OAAO,gBAAgB,CAAC,SAAS,CAAC,KAAK,SAAS,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,MAAM,IAAI,GAAG,mBAAmB,EAAE,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,UAAU,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IACnD,OAAO,sBAAsB,CAAC,UAAU,CAAC,KAAK,SAAS,CAAC;AAC1D,CAAC"}

package/dist/pipeline/tagger.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Replacement Tagger
+ * Replaces PII spans with placeholder tags and builds the PII map
+ */
+import { PIIType, SpanMatch, DetectedEntity, AnonymizationPolicy, SemanticAttributes } from "../types/index.js";
+/**
+ * PII Map entry (before encryption)
+ */
+export interface PIIMapEntry {
+    /** PII type */
+    type: PIIType;
+    /** Entity ID */
+    id: number;
+    /** Original text */
+    original: string;
+}
+/**
+ * Raw PII Map (before encryption)
+ */
+export type RawPIIMap = Map<string, string>;
+/**
+ * Tagging result
+ */
+export interface TaggingResult {
+    /** Anonymized text with placeholder tags */
+    anonymizedText: string;
+    /** List of detected entities with assigned IDs */
+    entities: DetectedEntity[];
+    /** Raw PII map (type_id -> original) */
+    piiMap: RawPIIMap;
+}
+/**
+ * Generates a PII placeholder tag
+ * Format: <PII type="TYPE" id="N"/> or <PII type="TYPE" gender="X" id="N"/> etc.
+ *
+ * Semantic attributes (gender, scope) are included when provided and not 'unknown'
+ */
+export declare function generateTag(type: PIIType, id: number, semantic?: SemanticAttributes): string;
+/**
+ * Result of parsing a PII tag
+ */
+export interface ParsedTag {
+    type: PIIType;
+    id: number;
+    semantic?: SemanticAttributes;
+}
+/**
+ * Parses a PII tag to extract type, id, and semantic attributes
+ * Returns null if not a valid tag
+ *
+ * Supports formats:
+ * - <PII type="TYPE" id="N"/>
+ * - <PII type="TYPE" gender="X" id="N"/>
+ * - <PII type="TYPE" scope="X" id="N"/>
+ * - <PII type="TYPE" gender="X" scope="Y" id="N"/>
+ */
+export declare function parseTag(tag: string): ParsedTag | null;
+/**
+ * Creates a key for the PII map
+ */
+export declare function createPIIMapKey(type: PIIType, id: number): string;
+/**
+ * Tags PII spans in text and builds the PII map
+ */
+export declare function tagEntities(text: string, matches: SpanMatch[], policy: AnonymizationPolicy): TaggingResult;
+/**
+ * Validates that a tag is well-formed
+ */
+export declare function isValidTag(tag: string): boolean;
+/**
+ * Tag extraction result with the matched text for accurate replacement
+ */
+export interface ExtractedTag {
+    type: PIIType;
+    id: number;
+    position: number;
+    /** The actual matched text (needed for replacement when tag is mangled) */
+    matchedText: string;
+    /** Semantic attributes extracted from the tag */
+    semantic?: SemanticAttributes;
+}
+/**
+ * Extracts all PII tags from anonymized text using fuzzy matching
+ * Handles mangled tags that may occur after translation
+ *
+ * Translation can mangle tags by:
+ * - Changing quote types (" → " or „ or « etc.)
+ * - Adding/removing whitespace
+ * - Changing case (type → Type, PII → pii)
+ * - Reordering attributes (id before type)
+ * - Modifying self-closing syntax (/> → / > or >)
+ */
+export declare function extractTags(anonymizedText: string): ExtractedTag[];
+/**
+ * Extracts tags using strict matching (original behavior)
+ * Useful when you know tags haven't been mangled
+ * Supports optional gender and scope attributes
+ */
+export declare function extractTagsStrict(anonymizedText: string): ExtractedTag[];
+/**
+ * Counts entities by type
+ */
+export declare function countEntitiesByType(entities: DetectedEntity[]): Record<PIIType, number>;
+/**
+ * Rehydrates anonymized text using the PII map
+ * Uses fuzzy matching to handle tags that may have been mangled by translation
+ *
+ * @param anonymizedText - Text containing PII tags (possibly mangled)
+ * @param piiMap - Map of PII keys to original values
+ * @param strict - If true, use strict matching (original behavior). Default: false
+ * @returns Text with PII tags replaced by original values
+ */
+export declare function rehydrate(anonymizedText: string, piiMap: RawPIIMap, strict?: boolean): string;
+//# sourceMappingURL=tagger.d.ts.map

package/dist/pipeline/tagger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tagger.d.ts","sourceRoot":"","sources":["../../src/pipeline/tagger.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,OAAO,EACP,SAAS,EACT,cAAc,EACd,mBAAmB,EACnB,kBAAkB,EACnB,MAAM,mBAAmB,CAAC;AAG3B;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,eAAe;IACf,IAAI,EAAE,OAAO,CAAC;IACd,gBAAgB;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,4CAA4C;IAC5C,cAAc,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,wCAAwC;IACxC,MAAM,EAAE,SAAS,CAAC;CACnB;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CACzB,IAAI,EAAE,OAAO,EACb,EAAE,EAAE,MAAM,EACV,QAAQ,CAAC,EAAE,kBAAkB,GAC5B,MAAM,CAcR;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,OAAO,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AAED;;;;;;;;;GASG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CA+CtD;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAEjE;AAED;;GAEG;AACH,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,SAAS,EAAE,EACpB,MAAM,EAAE,mBAAmB,GAC1B,aAAa,CA8Ef;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAE/C;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,WAAW,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,QAAQ,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AA4DD;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CAAC,cAAc,EAAE,MAAM,GAAG,YAAY,EAAE,CAyFlE;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,YAAY,EAAE,CAqDxE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,cAAc,EAAE,GACzB,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAczB;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CACvB,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,SAAS,EACjB,MAAM,GAAE,OAAe,GACtB,MAAM,CAyBR"}