pdf-anonymizer-core 0.3.0__tar.gz

pdf_anonymizer_core-0.3.0/PKG-INFO

@@ -0,0 +1,90 @@
+Metadata-Version: 2.4
+Name: pdf-anonymizer-core
+Version: 0.3.0
+Summary: A core library to anonymize PDF, Markdown, and plain text files using LLMs.
+Author-email: Leonid Ganeline <leo.gan.57@gmail.com>
+License: MIT
+Project-URL: repository, https://github.com/leo-gan/anonymizer
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: python-dotenv
+Requires-Dist: pymupdf4llm
+Requires-Dist: langchain-text-splitters
+Requires-Dist: google-genai
+Requires-Dist: ollama
+
+# PDF Anonymizer Core
+
+This package provides the core functionality for the PDF/Text anonymizer, including text extraction, LLM-driven anonymization, and deanonymization logic. It is used by `pdf-anonymizer-cli`.
+
+## Installation for Development
+
+This project uses `uv` and is structured as a monorepo. To install the necessary dependencies for development, run the following command from the root of the repository:
+
+```bash
+# From the repository root
+uv sync
+```
+
+This will install the `pdf-anonymizer-core` package in editable mode.
+
+## Environment Variables
+
+The core library itself does not load `.env` files. Environment variables must be loaded by the application that uses this library (e.g., `pdf-anonymizer-cli`) or set in your shell.
+
+- `GOOGLE_API_KEY`: Required when using Google's Gemini models.
+- `OLLAMA_HOST`: Optional, defaults to `http://localhost:11434` when using local Ollama models.
+
+## API Usage
+
+### `anonymize_file()`
+
+Anonymizes a single file and returns the anonymized text and a mapping of original entities to their placeholders.
+
+```python
+from pdf_anonymizer_core.core import anonymize_file
+from pdf_anonymizer_core.prompts import detailed
+
+# Example of programmatic usage
+text, mapping = anonymize_file(
+    file_path="/path/to/file.pdf",
+    prompt_template=detailed.prompt_template,
+    model_name="gemini-2.5-flash"
+)
+
+if text and mapping:
+    print("Anonymized Text:", text)
+    print("Mapping:", mapping)
+```
+
+### `deanonymize_file()`
+
+Reverts anonymization using a mapping file.
+
+```python
+from pdf_anonymizer_core.utils import deanonymize_file
+
+# Assumes you have an anonymized file and a mapping file
+deanonymized_text, stats = deanonymize_file(
+    anonymized_file="path/to/anonymized.md",
+    mapping_file="path/to/mapping.json"
+)
+
+if deanonymized_text:
+    print("Deanonymized Text:", deanonymized_text)
+```
+
+### Configuration
+
+You can import default configurations and available models from the `conf` module.
+
+```python
+from pdf_anonymizer_core.conf import (
+    DEFAULT_MODEL_NAME,
+    ModelName,
+    PromptEnum,
+)
+
+print(f"Default model: {DEFAULT_MODEL_NAME}")
+print(f"Available Google models: {[m.value for m in ModelName if 'gemini' in m.value]}")
+```

pdf_anonymizer_core-0.3.0/README.md

@@ -0,0 +1,75 @@
+# PDF Anonymizer Core
+
+This package provides the core functionality for the PDF/Text anonymizer, including text extraction, LLM-driven anonymization, and deanonymization logic. It is used by `pdf-anonymizer-cli`.
+
+## Installation for Development
+
+This project uses `uv` and is structured as a monorepo. To install the necessary dependencies for development, run the following command from the root of the repository:
+
+```bash
+# From the repository root
+uv sync
+```
+
+This will install the `pdf-anonymizer-core` package in editable mode.
+
+## Environment Variables
+
+The core library itself does not load `.env` files. Environment variables must be loaded by the application that uses this library (e.g., `pdf-anonymizer-cli`) or set in your shell.
+
+- `GOOGLE_API_KEY`: Required when using Google's Gemini models.
+- `OLLAMA_HOST`: Optional, defaults to `http://localhost:11434` when using local Ollama models.
+
+## API Usage
+
+### `anonymize_file()`
+
+Anonymizes a single file and returns the anonymized text and a mapping of original entities to their placeholders.
+
+```python
+from pdf_anonymizer_core.core import anonymize_file
+from pdf_anonymizer_core.prompts import detailed
+
+# Example of programmatic usage
+text, mapping = anonymize_file(
+    file_path="/path/to/file.pdf",
+    prompt_template=detailed.prompt_template,
+    model_name="gemini-2.5-flash"
+)
+
+if text and mapping:
+    print("Anonymized Text:", text)
+    print("Mapping:", mapping)
+```
+
+### `deanonymize_file()`
+
+Reverts anonymization using a mapping file.
+
+```python
+from pdf_anonymizer_core.utils import deanonymize_file
+
+# Assumes you have an anonymized file and a mapping file
+deanonymized_text, stats = deanonymize_file(
+    anonymized_file="path/to/anonymized.md",
+    mapping_file="path/to/mapping.json"
+)
+
+if deanonymized_text:
+    print("Deanonymized Text:", deanonymized_text)
+```
+
+### Configuration
+
+You can import default configurations and available models from the `conf` module.
+
+```python
+from pdf_anonymizer_core.conf import (
+    DEFAULT_MODEL_NAME,
+    ModelName,
+    PromptEnum,
+)
+
+print(f"Default model: {DEFAULT_MODEL_NAME}")
+print(f"Available Google models: {[m.value for m in ModelName if 'gemini' in m.value]}")
+```

pdf_anonymizer_core-0.3.0/pyproject.toml

@@ -0,0 +1,25 @@
+[project]
+name = "pdf-anonymizer-core"
+version = "0.3.0"
+description = "A core library to anonymize PDF, Markdown, and plain text files using LLMs."
+authors = [{ name = "Leonid Ganeline", email = "leo.gan.57@gmail.com" }]
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "python-dotenv",
+    "pymupdf4llm",
+    "langchain-text-splitters",
+    "google-genai",
+    "ollama",
+]
+
+[project.urls]
+repository = "https://github.com/leo-gan/anonymizer"
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+package-dir = {"" = "src"}

pdf_anonymizer_core-0.3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core/call_llm.py

@@ -0,0 +1,119 @@
+import json
+import logging
+import time
+from typing import Dict, List, TypedDict, Union
+
+import ollama
+from google import genai
+
+from pdf_anonymizer_core.conf import ModelName, ModelProvider
+
+
+# Type definitions for better code clarity
+class OllamaResponse(TypedDict):
+    message: Dict[str, str]
+
+
+class ModelResponse(TypedDict):
+    text: str
+
+
+class Entity(TypedDict):
+    text: str
+    type: str
+    base_form: str
+
+
+class IdentificationResult(TypedDict):
+    entities: List[Entity]
+
+
+def identify_entities_with_llm(
+    text: str,
+    prompt_template: str,
+    model_name: str,
+) -> List[Entity]:
+    """
+    Identifies PII entities in a text chunk using a specified language model.
+    It retries on failure up to a maximum of 3 times.
+
+    Args:
+        text: The text to analyze.
+        prompt_template: The prompt template for the identification task.
+        model_name: The name of the model to use.
+
+    Returns:
+        A list of identified entities.
+    """
+    prompt = prompt_template.format(text=text)
+
+    response: Union[OllamaResponse, ModelResponse, None] = None
+    max_retries = 3
+
+    for attempt in range(max_retries):
+        try:
+            logging.info(
+                f"Calling '{model_name}': text: {len(text):,}, attempt {attempt + 1}"
+            )
+            model_enum = ModelName(model_name)
+
+            if model_enum.provider == ModelProvider.OLLAMA:
+                response = ollama.chat(
+                    model=model_name,
+                    messages=[{"role": "user", "content": prompt}],
+                )
+                raw_text: str = response["message"]["content"]
+            else:
+                client = genai.Client()
+                response = client.models.generate_content(
+                    model=model_name, contents=prompt
+                )
+                raw_text = response.text
+
+            cleaned_response = (
+                raw_text.strip().replace("```json", "").replace("```", "").strip()
+            )
+            result: IdentificationResult = json.loads(cleaned_response)
+
+            return result.get("entities", [])
+
+        except json.JSONDecodeError as e:
+            response_text = _get_response_text(response, model_name)
+            logging.error(
+                f"Attempt {attempt + 1} failed with JSON decode error: {e}, "
+                f"response: {response_text[:200]}..."
+            )
+            if attempt + 1 == max_retries:
+                logging.error("Max retries reached. Returning empty list.")
+                return []
+
+        except Exception as e:
+            logging.error(f"Attempt {attempt + 1} failed with an error: {e}")
+            if attempt + 1 == max_retries:
+                logging.error("Max retries reached. Returning empty list.")
+                return []
+
+        time.sleep(1)  # Wait before retrying
+
+    return []
+
+
+def _get_response_text(
+    response: Union[OllamaResponse, ModelResponse, None], model_name: str
+) -> str:
+    """Extract text content from different response types."""
+    if not response:
+        return ""
+
+    model_enum = ModelName(model_name)
+    if model_enum.provider == ModelProvider.OLLAMA:
+        if (
+            isinstance(response, dict)
+            and "message" in response
+            and "content" in response["message"]
+        ):
+            return response["message"]["content"]
+    elif hasattr(response, "text"):
+        return response.text
+
+    return ""

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core/conf.py

@@ -0,0 +1,57 @@
+from enum import Enum
+from typing import Type, TypeVar
+
+# Default values
+DEFAULT_CHARACTERS_TO_ANONYMIZE: int = 100000
+DEFAULT_PROMPT_NAME: str = "detailed"
+DEFAULT_MODEL_NAME: str = "gemini-2.5-flash"
+
+# Type variable for enum values
+T = TypeVar("T", bound=Enum)
+
+
+# Enum for prompt names
+class PromptEnum(str, Enum):
+    simple = "simple"
+    detailed = "detailed"
+
+
+class ModelProvider(str, Enum):
+    GOOGLE = "google"
+    OLLAMA = "ollama"
+
+
+# Then you could associate a provider with each model, for instance:
+class ModelName(str, Enum):
+    gemini_2_5_pro = "gemini-2.5-pro"
+    gemini_2_5_flash = "gemini-2.5-flash"
+    gemini_2_5_flash_lite = "gemini-2.5-flash-lite"
+    gemma = "gemma:7b"
+    phi = "phi4-mini"
+
+    @property
+    def provider(self) -> "ModelProvider":
+        if "gemini" in self.value:
+            return ModelProvider.GOOGLE
+        return ModelProvider.OLLAMA
+
+
+def get_enum_value(enum_type: Type[T], value: str) -> T:
+    """Safely get an enum value from a string.
+
+    Args:
+        enum_type: The enum class to get the value from.
+        value: The string value to look up in the enum.
+
+    Returns:
+        The corresponding enum member.
+
+    Raises:
+        ValueError: If the value is not found in the enum.
+    """
+    try:
+        return enum_type(value)
+    except ValueError as e:
+        raise ValueError(
+            f"Invalid value '{value}' for enum {enum_type.__name__}"
+        ) from e

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core/core.py

@@ -0,0 +1,142 @@
+import logging
+import os
+import time
+from typing import Dict, List, Optional, Tuple
+
+from pdf_anonymizer_core.call_llm import identify_entities_with_llm
+from pdf_anonymizer_core.load_and_extract import load_and_extract_text_from_file
+
+
+def anonymize_file(
+    file_path: str,
+    characters_to_anonymize: int,
+    prompt_template: str,
+    model_name: str,
+    anonymized_entities: Optional[List[str]] = None,
+) -> Tuple[Optional[str], Optional[Dict[str, str]]]:
+    """
+    Anonymize a file by processing its text content.
+
+    Args:
+        file_path: Path to the file to anonymize.
+        characters_to_anonymize: Number of characters to process in each chunk.
+        prompt_template: Template string for the anonymization prompt.
+        model_name: Name of the language model to use for anonymization.
+        anonymized_entities: A list of entities to anonymize.
+
+    Returns:
+        A tuple containing the anonymized text and the mapping of original to anonymized entities,
+        or (None, None) if processing fails.
+    """
+    # File: chunk and convert to text
+    file_size = os.path.getsize(file_path)
+    text_pages: List[str] = load_and_extract_text_from_file(
+        file_path, characters_to_anonymize
+    )
+
+    if not text_pages:
+        logging.warning("No text could be extracted from the file.")
+        return None, None
+
+    logging.info(f"Extracted text pages: {text_pages[0][:50]} ...")
+    extracted_text_size = sum(len(page) for page in text_pages)
+
+    logging.info(f"  - File size: {file_size / 1024:.2f} KB")
+    logging.info(f"  - Extracted text size: {extracted_text_size / 1024:.2f} KB")
+
+    # Anonymization:
+    anonymized_chunks: List[str] = []
+    final_mapping: Dict[str, str] = {}
+    placeholder_counts: Dict[str, int] = {}
+    base_entity_placeholders: Dict[str, str] = {}
+    variation_counters: Dict[str, int] = {}
+
+    for i, text_page in enumerate(text_pages):
+        logging.info(f"Identifying entities in part {i + 1}/{len(text_pages)}...")
+        start_time = time.time()
+
+        all_entities = identify_entities_with_llm(
+            text_page, prompt_template, model_name
+        )
+
+        end_time = time.time()
+        duration = end_time - start_time
+        minutes = int(duration // 60)
+        seconds = int(duration % 60)
+        logging.info(f"   LLM call duration: {minutes}:{seconds:02d}")
+
+        entities_to_process = all_entities
+        if anonymized_entities:
+            entities_to_process = [
+                e for e in all_entities if e["type"] in anonymized_entities
+            ]
+
+        logging.info(
+            f"Found {len(all_entities)} total entities. "
+            f"Processing {len(entities_to_process)} entities."
+        )
+
+        # Consolidate base forms to handle variations like "John" vs "John Doe"
+        base_forms = {
+            e.get("base_form") for e in entities_to_process if e.get("base_form")
+        }
+        sorted_base_forms = sorted(list(base_forms), key=len, reverse=True)
+        for entity in entities_to_process:
+            base_form = entity.get("base_form")
+            if not base_form:
+                continue
+            for potential_full_form in sorted_base_forms:
+                if (
+                    base_form != potential_full_form
+                    and base_form in potential_full_form
+                ):
+                    entity["base_form"] = potential_full_form
+                    break
+
+        # Generate placeholders for all entities that need to be processed
+        for entity in entities_to_process:
+            entity_text = entity["text"]
+            entity_type = entity["type"].upper()
+            base_form = entity.get("base_form") or entity_text
+
+            if entity_text in final_mapping:
+                continue
+
+            if base_form not in base_entity_placeholders:
+                # New base entity, create main placeholder
+                current_count = placeholder_counts.get(entity_type, 0) + 1
+                placeholder_counts[entity_type] = current_count
+                main_placeholder = f"{entity_type}_{current_count}"
+                base_entity_placeholders[base_form] = main_placeholder
+                if base_form not in final_mapping:
+                    final_mapping[base_form] = main_placeholder
+
+            main_placeholder = base_entity_placeholders[base_form]
+
+            if entity_text != base_form:
+                # It's a variation, create variation placeholder
+                current_variation_count = (
+                    variation_counters.get(main_placeholder, 0) + 1
+                )
+                variation_counters[main_placeholder] = current_variation_count
+                variation_placeholder = (
+                    f"{main_placeholder}.v_{current_variation_count}"
+                )
+                final_mapping[entity_text] = variation_placeholder
+            else:
+                final_mapping[entity_text] = main_placeholder
+
+        # Sort entities by length descending to replace longer strings first
+        entities_to_process.sort(key=lambda e: len(e["text"]), reverse=True)
+
+        anonymized_text = text_page
+        for entity in entities_to_process:
+            placeholder = final_mapping.get(entity["text"])
+            if placeholder:
+                anonymized_text = anonymized_text.replace(entity["text"], placeholder)
+
+        anonymized_chunks.append(anonymized_text)
+
+    full_anonymized_text = "\n\n--- Page Break ---\n\n".join(anonymized_chunks)
+
+    return full_anonymized_text, final_mapping

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core/load_and_extract.py

@@ -0,0 +1,90 @@
+import logging
+from pathlib import Path
+
+import pymupdf4llm
+from langchain_text_splitters import (
+    MarkdownTextSplitter,
+    RecursiveCharacterTextSplitter,
+)
+
+
+def load_and_extract_text_from_pdf(
+    file_path: str, characters_to_anonymize: int = 100000
+) -> list[str]:
+    """
+    Loads a PDF file and extracts text from each page.
+
+    Args:
+        characters_to_anonymize: Number of characters to anonymize in one go.
+        file_path (str): The path to the PDF file.
+
+    Returns:
+        list: A list of strings, where each string is the text of a page.
+    """
+    try:
+        md_text = pymupdf4llm.to_markdown(file_path, show_progress=False)
+        splitter = MarkdownTextSplitter(
+            chunk_size=characters_to_anonymize, chunk_overlap=0
+        )
+        docs = splitter.create_documents([md_text])
+        return [doc.page_content for doc in docs]
+    except FileNotFoundError as e:
+        logging.error(f"Error: The file at {file_path} was not found.")
+        raise e
+    except Exception as e:
+        logging.error(f"An error occurred while reading the PDF: {e}")
+        raise e
+
+
+def load_and_extract_text_from_file(
+    file_path: str, characters_to_anonymize: int = 100000
+) -> list[str]:
+    """
+    Loads a file and extracts text, splitting it into chunks.
+
+    Args:
+        file_path (str): The path to the file.
+        characters_to_anonymize: Number of characters to process in each chunk.
+
+    Returns:
+        list: A list of strings, where each string is a chunk of text.
+    """
+    path = Path(file_path)
+    file_extension = path.suffix.lower()
+
+    try:
+        if file_extension == ".pdf":
+            return load_and_extract_text_from_pdf(file_path, characters_to_anonymize)
+        elif file_extension == ".md":
+            with open(file_path, "r", encoding="utf-8") as f:
+                text = f.read()
+            splitter = MarkdownTextSplitter(
+                chunk_size=characters_to_anonymize, chunk_overlap=0
+            )
+            docs = splitter.create_documents([text])
+            return [doc.page_content for doc in docs]
+        elif file_extension == ".txt":
+            with open(file_path, "r", encoding="utf-8") as f:
+                text = f.read()
+            splitter = RecursiveCharacterTextSplitter(
+                chunk_size=characters_to_anonymize, chunk_overlap=0
+            )
+            docs = splitter.create_documents([text])
+            return [doc.page_content for doc in docs]
+        else:
+            logging.warning(
+                f"Unsupported file type: {file_extension}. Treating as plain text."
+            )
+            with open(file_path, "r", encoding="utf-8") as f:
+                text = f.read()
+            splitter = RecursiveCharacterTextSplitter(
+                chunk_size=characters_to_anonymize, chunk_overlap=0
+            )
+            docs = splitter.create_documents([text])
+            return [doc.page_content for doc in docs]
+    except FileNotFoundError as e:
+        logging.error(f"Error: The file at {file_path} was not found.")
+        raise e
+    except Exception as e:
+        logging.error(f"An error occurred while reading the file: {e}")
+        raise e

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core/prompts/detailed.py

@@ -0,0 +1,55 @@
+prompt_template = """
+    You are an expert in identifying Personally Identifiable Information (PII) with high accuracy and contextual understanding.
+    Your task is to read the text below and identify all PII entities, including their variations.
+
+    Instructions:
+    1.  **Read the text carefully** to understand the context.
+    2.  **Identify all PII** based on the guidelines below.
+    3.  **Use contextual awareness.** For example, "Apple" as a fruit should not be identified, but "Apple Inc." as a company should.
+    4.  **Handle variations.** For each entity identified, you must determine its base form. For example, the base form of "Mary's" is "Mary Smith" if the context refers to a person named Mary Smith. The base form of "Mr. John Doe" is "John Doe".
+    5.  **Return a single JSON object** with one key: "entities".
+    6.  The value of "entities" should be a list of JSON objects. Each object represents a PII entity and MUST have the following keys:
+        - "text": The exact PII text found in the document.
+        - "type": The type of the entity (e.g., PERSON, ORGANIZATION).
+        - "base_form": The canonical or base form of the entity.
+
+    ENTITY TYPES:
+    *   **PERSON:** Full names, first names, last names, middle names, and their variations (e.g., possessives, titles).
+    *   **ADDRESS:** Street names, house numbers, city names, state/province names, postal codes, country names.
+    *   **DATE:** Only birthdates. Do not identify other dates.
+    *   **PHONE:** Any numerical sequences resembling phone numbers.
+    *   **EMAIL:** Standard email address formats.
+    *   **ORGANIZATION:** Names of organizations, businesses, companies.
+    *   **JOB_TITLE:** Specific roles or positions within organizations.
+    *   **ID:** Any alphanumeric strings that appear to be account numbers or identifiers.
+    *   **LOCATION:** Locations that are not full addresses, like cities or landmarks.
+
+    Example 1:
+    Text: "Mr. John Doe from Acme Inc. visited our office in Springfield yesterday. We discussed Mary's project."
+    Response:
+    {{
+        "entities": [
+            {{"text": "Mr. John Doe", "type": "PERSON", "base_form": "John Doe"}},
+            {{"text": "Acme Inc.", "type": "ORGANIZATION", "base_form": "Acme Inc."}},
+            {{"text": "Springfield", "type": "LOCATION", "base_form": "Springfield"}},
+            {{"text": "Mary's", "type": "PERSON", "base_form": "Mary"}}
+        ]
+    }}
+
+    Example 2:
+    Text: "We need to review John's latest report about the project for The New York Times."
+    Response:
+    {{
+        "entities": [
+            {{"text": "John's", "type": "PERSON", "base_form": "John"}},
+            {{"text": "The New York Times", "type": "ORGANIZATION", "base_form": "The New York Times"}}
+        ]
+    }}
+
+    Text to process:
+    ---
+    {text}
+    ---
+
+    Respond with ONLY the JSON object.
+    """

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core/prompts/simple.py

@@ -0,0 +1,27 @@
+prompt_template = """
+    You are an expert in identifying Personally Identifiable Information (PII).
+    Your task is to read the text below and identify all PII entities.
+
+    Instructions:
+    1.  Read the text carefully.
+    2.  Identify all PII, such as names, locations, organizations, phone numbers, email addresses, etc.
+    3.  Return a single JSON object with one key: "entities".
+    4.  The value of "entities" should be a list of JSON objects, where each object represents a PII entity and has two keys: "text" (the PII) and "type" (the entity type, e.g., PERSON, ORGANIZATION).
+
+    Example:
+    Text: "John Doe from Acme Inc. visited our office."
+    Response:
+    {{
+        "entities": [
+            {{"text": "John Doe", "type": "PERSON"}},
+            {{"text": "Acme Inc.", "type": "ORGANIZATION"}}
+        ]
+    }}
+
+    Text to process:
+    ---
+    {text}
+    ---
+
+    Respond with ONLY the JSON object.
+    """

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core/utils.py

@@ -0,0 +1,194 @@
+import json
+import os
+import re
+from pathlib import Path
+from typing import Dict, Tuple
+
+_PLACEHOLDER_PATTERN = re.compile(r"^[A-Z_]+_[0-9]+(?:\.v_[0-9]+)?$")
+
+
+def consolidate_mapping(
+    anonymized_text: str, mapping: Dict[str, str]
+) -> Tuple[str, Dict[str, str]]:
+    """
+    Consolidates the mapping to ensure one-to-one correspondence and updates the text.
+
+    Args:
+        anonymized_text: The text with anonymized placeholders.
+        mapping: The dictionary mapping placeholders to original PII.
+
+    Returns:
+        A tuple containing the updated anonymized text and the consolidated mapping.
+    """
+    # Invert the mapping to find duplicates
+    value_to_keys: Dict[str, list] = {}
+    for key, value in mapping.items():
+        if value not in value_to_keys:
+            value_to_keys[value] = []
+        value_to_keys[value].append(key)
+
+    consolidation_map = {}
+    consolidated_mapping = mapping.copy()
+
+    for value, keys in value_to_keys.items():
+        if len(keys) > 1:
+            canonical_key = keys[0]
+            for key_to_replace in keys[1:]:
+                consolidation_map[key_to_replace] = canonical_key
+                if key_to_replace in consolidated_mapping:
+                    del consolidated_mapping[key_to_replace]
+
+    # Update the anonymized text
+    for old_key, new_key in consolidation_map.items():
+        # Use word boundaries to avoid replacing parts of other words
+        anonymized_text = re.sub(
+            r"\b" + re.escape(old_key) + r"\b", new_key, anonymized_text
+        )
+
+    return anonymized_text, consolidated_mapping
+
+
+def save_results(
+    full_anonymized_text: str, final_mapping: dict[str, str], file_path: str
+) -> tuple[str, str]:
+    """
+    Save the anonymized text and the mapping to files.
+
+    Args:
+        full_anonymized_text (str): The anonymized text.
+        final_mapping (dict[str, str]): Mapping of original text -> placeholder.
+        file_path (str): The path to the original file.
+
+    Returns:
+        tuple[str, str]: The paths to the anonymized text file and the mapping file.
+    """
+    original_path = Path(file_path)
+    file_stem = original_path.stem
+    file_extension = original_path.suffix.lower()
+
+    anonymized_dir = "data/anonymized"
+    mappings_dir = "data/mappings"
+    os.makedirs(anonymized_dir, exist_ok=True)
+    os.makedirs(mappings_dir, exist_ok=True)
+
+    if file_extension == ".pdf":
+        output_extension = ".md"
+    else:
+        output_extension = file_extension
+
+    anonymized_output_file = (
+        f"{anonymized_dir}/{file_stem}.anonymized{output_extension}"
+    )
+    with open(anonymized_output_file, "w", encoding="utf-8") as f:
+        f.write(full_anonymized_text)
+
+    mapping_file = f"{mappings_dir}/{file_stem}.mapping.json"
+    # Persist mapping as placeholder -> original for correct deanonymization
+    with open(mapping_file, "w", encoding="utf-8") as f:
+        json.dump(final_mapping, f, indent=4)
+
+    return anonymized_output_file, mapping_file
+
+
+def deanonymize_file(
+    anonymized_file_path: str, mapping_file_path: str
+) -> tuple[str, str]:
+    """
+    Deanonymize a file using a mapping file.
+
+    The mapping file can be either:
+    - placeholder -> original (preferred), or
+    - original -> placeholder (legacy). In this case it will be inverted.
+
+    Variations like "PERSON_1.v_1" will be mapped to the base placeholder's
+    original value if only the base (e.g., "PERSON_1") exists in the mapping.
+
+    Args:
+        anonymized_file_path (str): Path to the anonymized file.
+        mapping_file_path (str): Path to the mapping file.
+
+    Returns:
+        A tuple containing the path to the deanonymized file and the statistics file.
+    """
+    with open(anonymized_file_path, "r", encoding="utf-8") as f:
+        anonymized_text = f.read()
+
+    with open(mapping_file_path, "r", encoding="utf-8") as f:
+        raw_mapping = json.load(f)
+
+    # Detect mapping direction and normalize to placeholder -> original
+    # Heuristic: if most keys look like placeholders (e.g., PERSON_1), treat as placeholder->original
+    placeholder_key_pattern = _PLACEHOLDER_PATTERN
+    keys_look_like_placeholders = sum(
+        1
+        for k in raw_mapping.keys()
+        if isinstance(k, str) and placeholder_key_pattern.match(k)
+    )
+    values_look_like_placeholders = sum(
+        1
+        for v in raw_mapping.values()
+        if isinstance(v, str) and placeholder_key_pattern.match(v)
+    )
+
+    if keys_look_like_placeholders >= values_look_like_placeholders:
+        placeholder_to_original = dict(raw_mapping)
+    else:
+        # Legacy: invert original -> placeholder to placeholder -> original
+        placeholder_to_original = {}
+        for original, placeholder in raw_mapping.items():
+            if isinstance(placeholder, str):
+                placeholder_to_original.setdefault(placeholder, original)
+
+    deanonymized_text = anonymized_text
+    used_placeholders = set()  # track actual placeholders (including variations) found
+
+    # Replace placeholders by longest first to avoid partial overlaps
+    sorted_placeholders = sorted(placeholder_to_original.keys(), key=len, reverse=True)
+
+    for base_placeholder in sorted_placeholders:
+        original_value = placeholder_to_original[base_placeholder]
+        # Match base and its variations: PERSON_1 and PERSON_1.v_1, PERSON_1.v_2, ...
+        pattern = re.compile(rf"\b{re.escape(base_placeholder)}(?:\.v_\d+)?\b")
+
+        # Record any matches before substitution
+        matches = set(pattern.findall(deanonymized_text))
+        if matches:
+            used_placeholders.update(matches)
+            deanonymized_text = pattern.sub(original_value, deanonymized_text)
+
+    # Gather stats
+    all_placeholders_in_text = set(
+        re.findall(r"[A-Z_]+_[0-9]+(?:\.v_[0-9]+)?", anonymized_text)
+    )
+
+    not_found_mappings = sorted(list(all_placeholders_in_text - used_placeholders))
+
+    # Unused mappings: base placeholders that never occurred (neither base nor any variation)
+    used_bases = {p.split(".v_")[0] for p in used_placeholders}
+    unused_mappings = sorted([p for p in sorted_placeholders if p not in used_bases])
+
+    anonymized_path = Path(anonymized_file_path)
+    file_stem = anonymized_path.name.replace(f".anonymized{anonymized_path.suffix}", "")
+    output_extension = anonymized_path.suffix
+
+    deanonymized_dir = "data/deanonymized"
+    stats_dir = "data/stats"
+    os.makedirs(deanonymized_dir, exist_ok=True)
+    os.makedirs(stats_dir, exist_ok=True)
+
+    deanonymized_file = f"{deanonymized_dir}/{file_stem}.deanonymized{output_extension}"
+    with open(deanonymized_file, "w", encoding="utf-8") as f:
+        f.write(deanonymized_text)
+
+    stats_file = f"{stats_dir}/{file_stem}.deanonymization_stat.json"
+    stats = {
+        "anonymized_file": anonymized_file_path,
+        "mapping_file": mapping_file_path,
+        "deanonymized_file": deanonymized_file,
+        "unused_mappings": unused_mappings,
+        "not_found_mappings": not_found_mappings,
+    }
+    with open(stats_file, "w", encoding="utf-8") as f:
+        json.dump(stats, f, indent=4)
+
+    return deanonymized_file, stats_file

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core.egg-info/PKG-INFO

@@ -0,0 +1,90 @@
+Metadata-Version: 2.4
+Name: pdf-anonymizer-core
+Version: 0.3.0
+Summary: A core library to anonymize PDF, Markdown, and plain text files using LLMs.
+Author-email: Leonid Ganeline <leo.gan.57@gmail.com>
+License: MIT
+Project-URL: repository, https://github.com/leo-gan/anonymizer
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: python-dotenv
+Requires-Dist: pymupdf4llm
+Requires-Dist: langchain-text-splitters
+Requires-Dist: google-genai
+Requires-Dist: ollama
+
+# PDF Anonymizer Core
+
+This package provides the core functionality for the PDF/Text anonymizer, including text extraction, LLM-driven anonymization, and deanonymization logic. It is used by `pdf-anonymizer-cli`.
+
+## Installation for Development
+
+This project uses `uv` and is structured as a monorepo. To install the necessary dependencies for development, run the following command from the root of the repository:
+
+```bash
+# From the repository root
+uv sync
+```
+
+This will install the `pdf-anonymizer-core` package in editable mode.
+
+## Environment Variables
+
+The core library itself does not load `.env` files. Environment variables must be loaded by the application that uses this library (e.g., `pdf-anonymizer-cli`) or set in your shell.
+
+- `GOOGLE_API_KEY`: Required when using Google's Gemini models.
+- `OLLAMA_HOST`: Optional, defaults to `http://localhost:11434` when using local Ollama models.
+
+## API Usage
+
+### `anonymize_file()`
+
+Anonymizes a single file and returns the anonymized text and a mapping of original entities to their placeholders.
+
+```python
+from pdf_anonymizer_core.core import anonymize_file
+from pdf_anonymizer_core.prompts import detailed
+
+# Example of programmatic usage
+text, mapping = anonymize_file(
+    file_path="/path/to/file.pdf",
+    prompt_template=detailed.prompt_template,
+    model_name="gemini-2.5-flash"
+)
+
+if text and mapping:
+    print("Anonymized Text:", text)
+    print("Mapping:", mapping)
+```
+
+### `deanonymize_file()`
+
+Reverts anonymization using a mapping file.
+
+```python
+from pdf_anonymizer_core.utils import deanonymize_file
+
+# Assumes you have an anonymized file and a mapping file
+deanonymized_text, stats = deanonymize_file(
+    anonymized_file="path/to/anonymized.md",
+    mapping_file="path/to/mapping.json"
+)
+
+if deanonymized_text:
+    print("Deanonymized Text:", deanonymized_text)
+```
+
+### Configuration
+
+You can import default configurations and available models from the `conf` module.
+
+```python
+from pdf_anonymizer_core.conf import (
+    DEFAULT_MODEL_NAME,
+    ModelName,
+    PromptEnum,
+)
+
+print(f"Default model: {DEFAULT_MODEL_NAME}")
+print(f"Available Google models: {[m.value for m in ModelName if 'gemini' in m.value]}")
+```

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+README.md
+pyproject.toml
+src/pdf_anonymizer_core/__init__.py
+src/pdf_anonymizer_core/call_llm.py
+src/pdf_anonymizer_core/conf.py
+src/pdf_anonymizer_core/core.py
+src/pdf_anonymizer_core/load_and_extract.py
+src/pdf_anonymizer_core/utils.py
+src/pdf_anonymizer_core.egg-info/PKG-INFO
+src/pdf_anonymizer_core.egg-info/SOURCES.txt
+src/pdf_anonymizer_core.egg-info/dependency_links.txt
+src/pdf_anonymizer_core.egg-info/requires.txt
+src/pdf_anonymizer_core.egg-info/top_level.txt
+src/pdf_anonymizer_core/prompts/__init__.py
+src/pdf_anonymizer_core/prompts/detailed.py
+src/pdf_anonymizer_core/prompts/simple.py

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core.egg-info/requires.txt

@@ -0,0 +1,5 @@
+python-dotenv
+pymupdf4llm
+langchain-text-splitters
+google-genai
+ollama

pdf_anonymizer_core-0.3.0/src/pdf_anonymizer_core.egg-info/top_level.txt

@@ -0,0 +1 @@
+pdf_anonymizer_core