presidio-structured 0.0.7__tar.gz

presidio_structured-0.0.7/PKG-INFO

@@ -0,0 +1,76 @@
+Metadata-Version: 2.4
+Name: presidio_structured
+Version: 0.0.7
+Summary: Presidio structured package - analyzes and anonymizes structured and semi-structured data.
+License-Expression: MIT
+Keywords: presidio_structured
+Author: Presidio
+Author-email: presidio@microsoft.com
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: click (>=8.1.0,<9.0.0)
+Requires-Dist: pandas (>=1.5.2,<4.0.0)
+Requires-Dist: presidio-analyzer (>=2.2.0,<3.0.0)
+Requires-Dist: presidio-anonymizer (>=2.2.0,<3.0.0)
+Project-URL: Homepage, https://github.com/data-privacy-stack/presidio
+Description-Content-Type: text/markdown
+
+# Presidio structured
+
+## Description
+
+The Presidio structured package is a flexible and customizable framework designed to identify and protect structured sensitive data. This tool extends the capabilities of Presidio, focusing on structured data formats such as tabular formats and semi-structured formats (JSON). It leverages the detection capabilities of Presidio-Analyzer to identify columns or keys containing personally identifiable information (PII), and establishes a mapping between these column/keys names and the detected PII entities. Following the detection, Presidio-Anonymizer is used to apply de-identification techniques to each value in columns identified as containing PII, ensuring the sensitive data is appropriately protected.
+
+## Installation
+
+### As a python package
+
+To install the `presidio-structured` package, run the following command:
+
+```sh
+pip install presidio-structured
+```
+
+### Getting started
+
+Anonymizing Data Frames:
+
+```py
+import pandas as pd
+from presidio_structured import StructuredEngine, PandasAnalysisBuilder
+from presidio_anonymizer.entities import OperatorConfig
+from faker import Faker # optionally using faker as an example
+
+# Initialize the engine with a Pandas data processor (default)
+pandas_engine = StructuredEngine()
+
+# Create a sample DataFrame
+sample_df = pd.DataFrame({'name': ['John Doe', 'Jane Smith'], 'email': ['john.doe@example.com', 'jane.smith@example.com']})
+
+# Generate a tabular analysis which describes PII entities in the DataFrame.
+tabular_analysis = PandasAnalysisBuilder().generate_analysis(sample_df)
+
+# Define anonymization operators
+fake = Faker()
+operators = {
+    "PERSON": OperatorConfig("replace", {"new_value": "REDACTED"}),
+    "EMAIL_ADDRESS": OperatorConfig("custom", {"lambda": lambda x: fake.safe_email()})
+}
+
+# Anonymize DataFrame
+anonymized_df = pandas_engine.anonymize(sample_df, tabular_analysis, operators=operators)
+print(anonymized_df)
+```
+
+## More information
+
+- [Docs](https://presidio.dataprivacystack.org/structured/)
+- [Samples](https://github.com/data-privacy-stack/presidio/blob/main/docs/samples/python/example_structured.ipynb)
+- [Join the discussion](https://github.com/data-privacy-stack/presidio/discussions?discussions_q=structured)
+- [Review issues on Github](https://github.com/data-privacy-stack/presidio/issues?q=is%3Aissue+label%3Astructured-data)
+

presidio_structured-0.0.7/README.md

@@ -0,0 +1,53 @@
+# Presidio structured
+
+## Description
+
+The Presidio structured package is a flexible and customizable framework designed to identify and protect structured sensitive data. This tool extends the capabilities of Presidio, focusing on structured data formats such as tabular formats and semi-structured formats (JSON). It leverages the detection capabilities of Presidio-Analyzer to identify columns or keys containing personally identifiable information (PII), and establishes a mapping between these column/keys names and the detected PII entities. Following the detection, Presidio-Anonymizer is used to apply de-identification techniques to each value in columns identified as containing PII, ensuring the sensitive data is appropriately protected.
+
+## Installation
+
+### As a python package
+
+To install the `presidio-structured` package, run the following command:
+
+```sh
+pip install presidio-structured
+```
+
+### Getting started
+
+Anonymizing Data Frames:
+
+```py
+import pandas as pd
+from presidio_structured import StructuredEngine, PandasAnalysisBuilder
+from presidio_anonymizer.entities import OperatorConfig
+from faker import Faker # optionally using faker as an example
+
+# Initialize the engine with a Pandas data processor (default)
+pandas_engine = StructuredEngine()
+
+# Create a sample DataFrame
+sample_df = pd.DataFrame({'name': ['John Doe', 'Jane Smith'], 'email': ['john.doe@example.com', 'jane.smith@example.com']})
+
+# Generate a tabular analysis which describes PII entities in the DataFrame.
+tabular_analysis = PandasAnalysisBuilder().generate_analysis(sample_df)
+
+# Define anonymization operators
+fake = Faker()
+operators = {
+    "PERSON": OperatorConfig("replace", {"new_value": "REDACTED"}),
+    "EMAIL_ADDRESS": OperatorConfig("custom", {"lambda": lambda x: fake.safe_email()})
+}
+
+# Anonymize DataFrame
+anonymized_df = pandas_engine.anonymize(sample_df, tabular_analysis, operators=operators)
+print(anonymized_df)
+```
+
+## More information
+
+- [Docs](https://presidio.dataprivacystack.org/structured/)
+- [Samples](https://github.com/data-privacy-stack/presidio/blob/main/docs/samples/python/example_structured.ipynb)
+- [Join the discussion](https://github.com/data-privacy-stack/presidio/discussions?discussions_q=structured)
+- [Review issues on Github](https://github.com/data-privacy-stack/presidio/issues?q=is%3Aissue+label%3Astructured-data)

presidio_structured-0.0.7/presidio_structured/__init__.py

@@ -0,0 +1,26 @@
+"""presidio-structured root module."""
+
+import logging
+
+from .analysis_builder import JsonAnalysisBuilder, PandasAnalysisBuilder
+from .config import StructuredAnalysis
+from .data import (
+    CsvReader,
+    JsonDataProcessor,
+    JsonReader,
+    PandasDataProcessor,
+)
+from .structured_engine import StructuredEngine
+
+logging.getLogger("presidio-structured").addHandler(logging.NullHandler())
+
+__all__ = [
+    "StructuredEngine",
+    "JsonAnalysisBuilder",
+    "PandasAnalysisBuilder",
+    "StructuredAnalysis",
+    "CsvReader",
+    "JsonReader",
+    "PandasDataProcessor",
+    "JsonDataProcessor",
+]

presidio_structured-0.0.7/presidio_structured/analysis_builder.py

@@ -0,0 +1,418 @@
+import logging
+from abc import ABC, abstractmethod
+from collections import Counter
+from collections.abc import Iterable
+from typing import Dict, Iterator, List, Optional, Union
+
+from pandas import DataFrame
+from presidio_analyzer import (
+    AnalyzerEngine,
+    BatchAnalyzerEngine,
+    DictAnalyzerResult,
+    RecognizerResult,
+)
+
+from presidio_structured.config import StructuredAnalysis
+
+NON_PII_ENTITY_TYPE = "NON_PII"
+
+logger = logging.getLogger("presidio-structured")
+
+
+class AnalysisBuilder(ABC):
+    """Abstract base class for a configuration generator."""
+
+    def __init__(
+        self,
+        analyzer: Optional[AnalyzerEngine] = None,
+        analyzer_score_threshold: Optional[float] = None,
+        n_process: int = 1,
+        batch_size: int = 1
+    ) -> None:
+        """Initialize the configuration generator.
+
+        :param analyzer: AnalyzerEngine instance
+        :param analyzer_score_threshold: threshold for filtering out results
+        :param batch_size: Batch size to process in a single iteration
+        :param n_process: Number of processors to use. Defaults to `1`
+        """
+        default_score_threshold = (
+            analyzer_score_threshold if analyzer_score_threshold is not None else 0
+        )
+        self.analyzer = (
+            AnalyzerEngine(default_score_threshold=default_score_threshold)
+            if analyzer is None
+            else analyzer
+        )
+        self.batch_analyzer = BatchAnalyzerEngine(analyzer_engine=self.analyzer)
+        self.n_process = n_process
+        self.batch_size = batch_size
+
+    @abstractmethod
+    def generate_analysis(
+        self,
+        data: Union[Dict, DataFrame],
+        language: str = "en",
+        score_threshold: Optional[float] = None,
+    ) -> StructuredAnalysis:
+        """
+        Abstract method to generate a configuration from the given data.
+
+        :param data: The input data. Can be a dictionary or DataFrame instance.
+        :return: The generated configuration.
+        """
+        pass
+
+    def _remove_low_scores(
+        self,
+        key_recognizer_result_map: Dict[str, RecognizerResult],
+        score_threshold: Optional[float] = None,
+    ) -> Dict[str, RecognizerResult]:
+        """
+        Remove results for which the confidence is lower than the threshold.
+
+        :param results: Dict of column names to RecognizerResult
+        :param score_threshold: float value for minimum possible confidence
+        :return: List[RecognizerResult]
+        """
+        if score_threshold is None:
+            score_threshold = self.analyzer.default_score_threshold
+
+        new_key_recognizer_result_map = {}
+        for column, result in key_recognizer_result_map.items():
+            if result.score >= score_threshold:
+                new_key_recognizer_result_map[column] = result
+
+        return new_key_recognizer_result_map
+
+
+class JsonAnalysisBuilder(AnalysisBuilder):
+    """Concrete configuration generator for JSON data."""
+
+    def generate_analysis(
+        self,
+        data: Dict,
+        language: str = "en",
+    ) -> StructuredAnalysis:
+        """
+        Generate a configuration from the given JSON data.
+
+        :param data: The input JSON data.
+        :return: The generated configuration.
+        """
+        logger.debug("Starting JSON BatchAnalyzer analysis")
+        analyzer_results = self.batch_analyzer.analyze_dict(
+            input_dict=data,
+            language=language,
+            n_process=self.n_process,
+            batch_size=self.batch_size
+        )
+
+        key_recognizer_result_map = self._generate_analysis_from_results_json(
+            analyzer_results
+        )
+
+        key_entity_map = {
+            key: result.entity_type for key, result in key_recognizer_result_map.items()
+        }
+
+        return StructuredAnalysis(entity_mapping=key_entity_map)
+
+    def _generate_analysis_from_results_json(
+        self, analyzer_results: Iterator[DictAnalyzerResult], prefix: str = ""
+    ) -> Dict[str, RecognizerResult]:
+        """
+        Generate a configuration from the given analyzer results. Always uses the first recognizer result if there are more than one.
+
+        :param analyzer_results: The analyzer results.
+        :param prefix: The prefix for the configuration keys.
+        :return: The generated configuration.
+        """  # noqa: E501
+        key_recognizer_result_map = {}
+
+        if not isinstance(analyzer_results, Iterable):
+            logger.debug(
+                "No analyzer results found, returning empty StructuredAnalysis"
+            )
+            return key_recognizer_result_map
+
+        for result in analyzer_results:
+            current_key = prefix + result.key
+
+            if isinstance(result.value, dict) and isinstance(
+                result.recognizer_results, Iterator
+            ):
+                nested_mappings = self._generate_analysis_from_results_json(
+                    result.recognizer_results, prefix=current_key + "."
+                )
+                key_recognizer_result_map.update(nested_mappings)
+            first_recognizer_result = next(iter(result.recognizer_results), None)
+            if isinstance(first_recognizer_result, RecognizerResult):
+                logger.debug(
+                    f"Found result with entity {first_recognizer_result.entity_type} \
+                        in {current_key}"
+                )
+                key_recognizer_result_map[current_key] = first_recognizer_result
+        return key_recognizer_result_map
+
+
+class TabularAnalysisBuilder(AnalysisBuilder):
+    """Placeholder class for generalizing tabular data analysis builders (e.g. PySpark). Only implemented as PandasAnalysisBuilder for now."""  # noqa: E501
+
+    pass
+
+
+class PandasAnalysisBuilder(TabularAnalysisBuilder):
+    """Concrete configuration generator for tabular data."""
+
+    entity_selection_strategies = {"highest_confidence", "mixed", "most_common"}
+
+    def generate_analysis(
+        self,
+        df: DataFrame,
+        n: Optional[int] = None,
+        language: str = "en",
+        selection_strategy: str = "most_common",
+        mixed_strategy_threshold: float = 0.5,
+    ) -> StructuredAnalysis:
+        """
+        Generate a configuration from the given tabular data.
+
+        :param df: The input tabular data (dataframe).
+        :param n: The number of samples to be taken from the dataframe.
+        :param language: The language to be used for analysis.
+        :param selection_strategy: A string that specifies the entity selection strategy
+        ('highest_confidence', 'mixed', or default to most common).
+        :param mixed_strategy_threshold: A float value for the threshold to be used in
+        the entity selection mixed strategy.
+        :return: A StructuredAnalysis object containing the analysis results.
+        """
+        if not n:
+            n = len(df)
+        elif n > len(df):
+            logger.debug(
+                f"Number of samples ({n}) is larger than the number of rows \
+                    ({len(df)}), using all rows"
+            )
+            n = len(df)
+
+        df = df.sample(n, random_state=123)
+
+        key_recognizer_result_map = self._generate_key_rec_results_map(
+            df, language, selection_strategy, mixed_strategy_threshold
+        )
+
+        key_entity_map = {
+            key: result.entity_type
+            for key, result in key_recognizer_result_map.items()
+            if result.entity_type != NON_PII_ENTITY_TYPE
+        }
+
+        return StructuredAnalysis(entity_mapping=key_entity_map)
+
+    def _generate_key_rec_results_map(
+        self,
+        df: DataFrame,
+        language: str,
+        selection_strategy: str = "most_common",
+        mixed_strategy_threshold: float = 0.5,
+    ) -> Dict[str, RecognizerResult]:
+        """
+        Find the most common entity in a dataframe column.
+
+        If more than one entity is found in a cell, the first one is used.
+
+        :param df: The dataframe where entities will be searched.
+        :param language: Language to be used in the analysis engine.
+        :param selection_strategy: A string that specifies the entity selection strategy
+        ('highest_confidence', 'mixed', or default to most common).
+        :param mixed_strategy_threshold: A float value for the threshold to be used in
+        the entity selection mixed strategy.
+        :return: A dictionary mapping column names to the most common RecognizerResult.
+        """
+        column_analyzer_results_map = self._batch_analyze_df(df, language)
+        key_recognizer_result_map = {}
+        for column, analyzer_result in column_analyzer_results_map.items():
+            key_recognizer_result_map[column] = self._find_entity_based_on_strategy(
+                analyzer_result, selection_strategy, mixed_strategy_threshold
+            )
+        return key_recognizer_result_map
+
+    def _batch_analyze_df(
+        self, df: DataFrame, language: str
+    ) -> Dict[str, List[List[RecognizerResult]]]:
+        """
+        Analyze each column in the dataframe for entities using the batch analyzer.
+
+        :param df: The dataframe to be analyzed.
+        :param language: The language configuration for the analyzer.
+        :return: A dictionary mapping each column name to a \
+            list of lists of RecognizerResults.
+        """
+        column_analyzer_results_map = {}
+        for column in df.columns:
+            logger.debug(f"Finding most common PII entity for column {column}")
+            analyzer_results = self.batch_analyzer.analyze_iterator(
+                [val for val in df[column]],
+                language=language,
+                n_process=self.n_process,
+                batch_size=self.batch_size
+            )
+            column_analyzer_results_map[column] = analyzer_results
+
+        return column_analyzer_results_map
+
+    def _find_entity_based_on_strategy(
+        self,
+        analyzer_results: List[List[RecognizerResult]],
+        selection_strategy: str,
+        mixed_strategy_threshold: float,
+    ) -> RecognizerResult:
+        """
+        Determine the most suitable entity based on the specified selection strategy.
+
+        :param analyzer_results: A nested list of RecognizerResult objects from the
+        analysis results.
+        :param selection_strategy: A string that specifies the entity selection strategy
+        ('highest_confidence', 'mixed', or default to most common).
+        :return: A RecognizerResult object representing the selected entity based on the
+        given strategy.
+        """
+        if selection_strategy not in self.entity_selection_strategies:
+            raise ValueError(
+                f"Unsupported entity selection strategy: {selection_strategy}."
+            )
+
+        if not any(analyzer_results):
+            return RecognizerResult(
+                entity_type=NON_PII_ENTITY_TYPE, start=0, end=1, score=1.0
+            )
+
+        flat_results = self._flatten_results(analyzer_results)
+
+        # Select the entity based on the desired strategy
+        if selection_strategy == "highest_confidence":
+            return self._select_highest_confidence_entity(flat_results)
+        elif selection_strategy == "mixed":
+            return self._select_mixed_strategy_entity(
+                flat_results, mixed_strategy_threshold
+            )
+
+        return self._select_most_common_entity(flat_results)
+
+    def _select_most_common_entity(self, flat_results):
+        """
+        Select the most common entity from the flattened analysis results.
+
+        :param flat_results: A list of tuples containing index and RecognizerResult
+        objects from the flattened analysis results.
+        :return: A RecognizerResult object for the most commonly found entity type.
+        """
+        # Count occurrences of each entity type
+        type_counter = Counter(res.entity_type for _, res in flat_results)
+        most_common_type, most_common_count = type_counter.most_common(1)[0]
+
+        # Calculate the score as the proportion of occurrences
+        score = most_common_count / len(flat_results)
+
+        return RecognizerResult(
+            entity_type=most_common_type, start=0, end=1, score=score
+        )
+
+    def _select_highest_confidence_entity(self, flat_results):
+        """
+        Select the entity with the highest confidence score.
+
+        :param flat_results: A list of tuples containing index and RecognizerResult
+        objects from the flattened analysis results.
+        :return: A RecognizerResult object for the entity with the highest confidence
+        score.
+        """
+        score_aggregator = self._aggregate_scores(flat_results)
+
+        # Find the highest score across all entities
+        highest_score = max(
+            max(scores) for scores in score_aggregator.values() if scores
+        )
+
+        # Find the entities with the highest score and count their occurrences
+        entities_highest_score = {
+            entity: scores.count(highest_score)
+            for entity, scores in score_aggregator.items()
+            if highest_score in scores
+        }
+
+        # Find the entity(ies) with the most number of high scores
+        max_occurrences = max(entities_highest_score.values())
+        highest_confidence_entities = [
+            entity
+            for entity, count in entities_highest_score.items()
+            if count == max_occurrences
+        ]
+
+        return RecognizerResult(
+            entity_type=highest_confidence_entities[0],
+            start=0,
+            end=1,
+            score=highest_score,
+        )
+
+    def _select_mixed_strategy_entity(self, flat_results, mixed_strategy_threshold):
+        """
+        Select an entity using a mixed strategy.
+
+        Chooses an entity based on the highest confidence score if it is above the
+        threshold. Otherwise, it defaults to the most common entity.
+
+        :param flat_results: A list of tuples containing index and RecognizerResult
+        objects from the flattened analysis results.
+        :return: A RecognizerResult object selected based on the mixed strategy.
+        """
+        # Check if mixed strategy threshold is within the valid range
+        if not 0 <= mixed_strategy_threshold <= 1:
+            raise ValueError(
+                f"Invalid mixed strategy threshold: {mixed_strategy_threshold}."
+            )
+
+        score_aggregator = self._aggregate_scores(flat_results)
+
+        # Check if the highest score is greater than threshold and select accordingly
+        highest_score = max(
+            max(scores) for scores in score_aggregator.values() if scores
+        )
+        if highest_score > mixed_strategy_threshold:
+            return self._select_highest_confidence_entity(flat_results)
+        else:
+            return self._select_most_common_entity(flat_results)
+
+    @staticmethod
+    def _aggregate_scores(flat_results):
+        """
+        Aggregate the scores for each entity type from the flattened analysis results.
+
+        :param flat_results: A list of tuples containing index and RecognizerResult
+        objects from the flattened analysis results.
+        :return: A dictionary with entity types as keys and lists of scores as values.
+        """
+        score_aggregator = {}
+        for _, res in flat_results:
+            if res.entity_type not in score_aggregator:
+                score_aggregator[res.entity_type] = []
+            score_aggregator[res.entity_type].append(res.score)
+        return score_aggregator
+
+    @staticmethod
+    def _flatten_results(analyzer_results):
+        """
+        Flattens a nested lists of RecognizerResult objects into a list of tuples.
+
+        :param analyzer_results: A nested list of RecognizerResult objects from
+        the analysis results.
+        :return: A flattened list of tuples containing index and RecognizerResult
+        objects.
+        """
+        return [
+            (cell_idx, res)
+            for cell_idx, cell_results in enumerate(analyzer_results)
+            for res in cell_results
+        ]

presidio_structured-0.0.7/presidio_structured/config/__init__.py

@@ -0,0 +1,7 @@
+"""Config module for presidio-structured."""
+
+from .structured_analysis import StructuredAnalysis
+
+__all__ = [
+    "StructuredAnalysis",
+]

presidio_structured-0.0.7/presidio_structured/config/structured_analysis.py

@@ -0,0 +1,20 @@
+"""Structured Analysis module."""
+
+from dataclasses import dataclass
+from typing import Dict
+
+
+@dataclass
+class StructuredAnalysis:
+    """
+    Dataclass containing entity analysis from structured data.
+
+    Currently, this class only contains entity mapping.
+
+    param entity_mapping : dict. Mapping column/key names to entity types, e.g., {
+        "person.name": "PERSON",
+        "person.address": "LOCATION"
+        }
+    """
+
+    entity_mapping: Dict[str, str]

presidio_structured-0.0.7/presidio_structured/data/__init__.py

@@ -0,0 +1,11 @@
+"""Data module."""
+
+from .data_processors import JsonDataProcessor, PandasDataProcessor
+from .data_reader import CsvReader, JsonReader
+
+__all__ = [
+    "CsvReader",
+    "JsonReader",
+    "PandasDataProcessor",
+    "JsonDataProcessor",
+]

presidio_structured-0.0.7/presidio_structured/data/data_processors.py

@@ -0,0 +1,222 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, List, Union
+
+from pandas import DataFrame
+from presidio_anonymizer.entities import OperatorConfig
+from presidio_anonymizer.operators import OperatorsFactory, OperatorType
+
+from presidio_structured.config import StructuredAnalysis
+
+
+class DataProcessorBase(ABC):
+    """Abstract class to handle logic of operations over text using the operators."""
+
+    def __init__(self) -> None:
+        """Initialize DataProcessorBase object."""
+        self.logger = logging.getLogger("presidio-structured")
+
+    def operate(
+        self,
+        data: Any,
+        structured_analysis: StructuredAnalysis,
+        operators: Dict[str, OperatorConfig],
+    ) -> Any:
+        """
+        Perform operations over the text using the operators, as per the structured analysis.
+
+        :param data: Data to be operated on.
+        :param structured_analysis: Analysis schema as per the structured data.
+        :param operators: Dictionary containing operator configuration objects.
+        :return: Data after being operated upon.
+        """  # noqa: E501
+        key_to_operator_mapping = self._generate_operator_mapping(
+            structured_analysis, operators
+        )
+        return self._process(data, key_to_operator_mapping)
+
+    @abstractmethod
+    def _process(
+        self,
+        data: Union[Dict, DataFrame],
+        key_to_operator_mapping: Dict[str, Callable],
+    ) -> Union[Dict, DataFrame]:
+        """
+        Abstract method for subclasses to provide operation implementation.
+
+        :param data: Data to be operated on.
+        :param key_to_operator_mapping: Mapping of keys to operators.
+        :return: Operated data.
+        """
+        pass
+
+    @staticmethod
+    def _create_operator_callable(operator, params):
+        def operator_callable(text):
+            return operator.operate(params=params, text=text)
+
+        return operator_callable
+
+    def _generate_operator_mapping(
+        self, config, operators: Dict[str, OperatorConfig]
+    ) -> Dict[str, Callable]:
+        """
+        Generate a mapping of keys to operator callables.
+
+        :param config: Configuration object containing mapping of entity types to keys.
+        :param operators: Dictionary containing operator configuration objects.
+        :return: Dictionary mapping keys to operator callables.
+        """
+        key_to_operator_mapping = {}
+
+        operators_factory = OperatorsFactory()
+        for key, entity in config.entity_mapping.items():
+            self.logger.debug(f"Creating operator for key {key} and entity {entity}")
+            operator_config = operators.get(entity, operators.get("DEFAULT", None))
+            if operator_config is None:
+                raise ValueError(f"Operator for entity {entity} not found")
+            # NOTE: hardcoded OperatorType.Anonymize, as this is the only one supported.
+            operator = operators_factory.create_operator_class(
+                operator_config.operator_name, OperatorType.Anonymize
+            )
+            operator_callable = self._create_operator_callable(
+                operator, operator_config.params
+            )
+            key_to_operator_mapping[key] = operator_callable
+
+        return key_to_operator_mapping
+
+    def _operate_on_text(
+        self,
+        text_to_operate_on: str,
+        operator_callable: Callable,
+    ) -> str:
+        """
+        Operates on the provided text using the operator callable.
+
+        :param text_to_operate_on: Text to be operated on.
+        :param operator_callable: Callable that performs operation on the text.
+        :return: Text after operation.
+        """
+        return operator_callable(text_to_operate_on)
+
+
+class PandasDataProcessor(DataProcessorBase):
+    """Pandas Data Processor."""
+
+    def _process(
+        self, data: DataFrame, key_to_operator_mapping: Dict[str, Callable]
+    ) -> DataFrame:
+        """
+        Operates on the given pandas DataFrame based on the provided operators.
+
+        :param data: DataFrame to be operated on.
+        :param key_to_operator_mapping: Mapping of keys to operator callables.
+        :return: DataFrame after the operation.
+        """
+
+        if not isinstance(data, DataFrame):
+            raise ValueError("Data must be a pandas DataFrame")
+
+        for key, operator_callable in key_to_operator_mapping.items():
+            self.logger.debug(f"Operating on column {key}")
+            for row in data.itertuples(index=True):
+                text_to_operate_on = getattr(row, key)
+                operated_text = self._operate_on_text(
+                    text_to_operate_on, operator_callable
+                )
+                data.at[row.Index, key] = operated_text
+        return data
+
+
+class JsonDataProcessor(DataProcessorBase):
+    """JSON Data Processor, Supports arbitrary nesting of dictionaries and lists."""
+
+    @staticmethod
+    def _get_nested_value(data: Union[Dict, List, None], path: List[str]) -> Any:
+        """
+        Recursively retrieves the value from nested data using a given path.
+
+        :param data: Nested data (list or dictionary).
+        :param path: List of keys/indexes representing the path.
+        :return: Retrieved value.
+        """
+        for i, key in enumerate(path):
+            if isinstance(data, list):
+                if key.isdigit():
+                    data = data[int(key)]
+                else:
+                    return [
+                        JsonDataProcessor._get_nested_value(item, path[i:])
+                        for item in data
+                    ]
+            elif isinstance(data, dict):
+                data = data.get(key)
+            else:
+                return data
+        return data
+
+    @staticmethod
+    def _set_nested_value(data: Union[Dict, List], path: List[str], value: Any) -> None:
+        """
+        Recursively sets a value in nested data using a given path.
+
+        :param data: Nested data (JSON-like).
+        :param path: List of keys/indexes representing the path.
+        :param value: Value to be set.
+        """
+        for i, key in enumerate(path):
+            if isinstance(data, list):
+                if i + 1 < len(path) and path[i + 1].isdigit():
+                    idx = int(path[i + 1])
+                    while len(data) <= idx:
+                        data.append({})
+                    data = data[idx]
+                    continue
+                else:
+                    for item in data:
+                        JsonDataProcessor._set_nested_value(item, path[i:], value)
+                    return
+            elif isinstance(data, dict):
+                if i == len(path) - 1:
+                    data[key] = value
+                else:
+                    data = data.setdefault(key, {})
+
+    def _process(
+        self,
+        data: Union[Dict, List],
+        key_to_operator_mapping: Dict[str, Callable],
+    ) -> Union[Dict, List]:
+        """
+        Operates on the given JSON-like data based on the provided configuration.
+
+        :param data: JSON-like data to be operated on.
+        :param key_to_operator_mapping: maps keys to Callable operators.
+        :return: JSON-like data after the operation.
+        """
+
+        if not isinstance(data, (dict, list)):
+            raise ValueError("Data must be a JSON-like object")
+
+        for key, operator_callable in key_to_operator_mapping.items():
+            self.logger.debug(f"Operating on key {key}")
+            keys = key.split(".")
+            if isinstance(data, list):
+                for item in data:
+                    self._process(item, key_to_operator_mapping)
+            else:
+                text_to_operate_on = self._get_nested_value(data, keys)
+                if text_to_operate_on:
+                    if isinstance(text_to_operate_on, list):
+                        for text in text_to_operate_on:
+                            operated_text = self._operate_on_text(
+                                text, operator_callable
+                            )
+                            self._set_nested_value(data, keys, operated_text)
+                    else:
+                        operated_text = self._operate_on_text(
+                            text_to_operate_on, operator_callable
+                        )
+                        self._set_nested_value(data, keys, operated_text)
+        return data

presidio_structured-0.0.7/presidio_structured/data/data_reader.py

@@ -0,0 +1,70 @@
+"""Helper data classes, mostly simple wrappers  to ensure consistent user interface."""
+
+import json
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, Union
+
+import pandas as pd
+
+
+class ReaderBase(ABC):
+    """
+    Base class for data readers.
+
+    This class should not be instantiated directly, instead init a subclass.
+    """
+
+    @abstractmethod
+    def read(self, path: Union[str, Path], **kwargs) -> Any:
+        """
+        Extract data from file located at path.
+
+        :param path: String defining the location of the file to read.
+        :return: The data read from the file.
+        """
+        pass
+
+
+class CsvReader(ReaderBase):
+    """
+    Reader for reading csv files.
+
+    Usage::
+
+        reader = CsvReader()
+        data = reader.read(path="filepath.csv")
+
+    """
+
+    def read(self, path: Union[str, Path], **kwargs) -> pd.DataFrame:
+        """
+        Read csv file to pandas dataframe.
+
+        :param path: String defining the location of the csv file to read.
+        :return: Pandas DataFrame with the data read from the csv file.
+        """
+        return pd.read_csv(path, **kwargs)
+
+
+class JsonReader(ReaderBase):
+    """
+    Reader for reading json files.
+
+    Usage::
+
+        reader = JsonReader()
+        data = reader.read(path="filepath.json")
+
+    """
+
+    def read(self, path: Union[str, Path], **kwargs) -> Dict[str, Any]:
+        """
+        Read json file to dict.
+
+        :param path: String defining the location of the json file to read.
+        :return: dictionary with the data read from the json file.
+        """
+        with open(path) as f:
+            data = json.load(f, **kwargs)
+        return data

presidio_structured-0.0.7/presidio_structured/py.typed

@@ -0,0 +1 @@
+# py.typed

presidio_structured-0.0.7/presidio_structured/structured_engine.py

@@ -0,0 +1,68 @@
+import logging
+from typing import Dict, Optional, Union
+
+from pandas import DataFrame
+from presidio_anonymizer.entities import OperatorConfig
+
+from presidio_structured.config import StructuredAnalysis
+from presidio_structured.data.data_processors import (
+    DataProcessorBase,
+    PandasDataProcessor,
+)
+
+DEFAULT = "replace"
+
+
+class StructuredEngine:
+    """Class to implement methods for anonymizing tabular data."""
+
+    def __init__(self, data_processor: Optional[DataProcessorBase] = None) -> None:
+        """
+        Initialize the class with a data processor.
+
+        :param data_processor: Instance of DataProcessorBase.
+        """
+        if data_processor is None:
+            self.data_processor = PandasDataProcessor()
+        else:
+            self.data_processor = data_processor
+
+        self.logger = logging.getLogger("presidio-structured")
+
+    def anonymize(
+        self,
+        data: Union[Dict, DataFrame],
+        structured_analysis: StructuredAnalysis,
+        operators: Union[Dict[str, OperatorConfig], None] = None,
+    ) -> Union[Dict, DataFrame]:
+        """
+        Anonymize the given data using the given configuration.
+
+        :param data: input data as dictionary or pandas DataFrame.
+        :param structured_analysis: structured analysis configuration.
+        :param operators: a dictionary of operator configurations, optional.
+        :return: Anonymized dictionary or DataFrame.
+        """
+        self.logger.debug("Starting anonymization")
+        operators = self.__check_or_add_default_operator(operators)
+
+        return self.data_processor.operate(data, structured_analysis, operators)
+
+    def __check_or_add_default_operator(
+        self, operators: Union[Dict[str, OperatorConfig], None]
+    ) -> Dict[str, OperatorConfig]:
+        """
+        Check if the provided operators dictionary has a default operator. If not, add a default operator.
+
+        :param operators: dictionary of operator configurations.
+        :return: operators dictionary with the default operator added \
+            if it was not initially present.
+        """  # noqa: E501
+        default_operator = OperatorConfig(DEFAULT)
+        if not operators:
+            self.logger.debug("No operators provided, using default operator")
+            return {"DEFAULT": default_operator}
+        if not operators.get("DEFAULT"):
+            self.logger.debug("No default operator provided, using default operator")
+            operators["DEFAULT"] = default_operator
+        return operators

presidio_structured-0.0.7/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core"]
+
+[project]
+name = "presidio_structured"
+version = "0.0.7"
+description = "Presidio structured package - analyzes and anonymizes structured and semi-structured data."
+authors = [{name = "Presidio", email = "presidio@microsoft.com"}]
+license = "MIT"
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+keywords = ["presidio_structured"]
+urls = {Homepage = "https://github.com/data-privacy-stack/presidio"}
+readme = "README.md"
+requires-python = ">=3.10,<4.0"
+
+dependencies = [
+    "presidio-analyzer (>=2.2.0,<3.0.0)",
+    "presidio-anonymizer (>=2.2.0,<3.0.0)",
+    "pandas (>=1.5.2,<4.0.0)",
+    "click (>=8.1.0,<9.0.0)",
+]
+
+[tool.poetry.group.dev.dependencies]
+pip = "*"
+ruff = "*"
+pytest = "*"
+pytest-cov = "*"
+pytest-mock = "*"
+python-dotenv = "*"
+pre_commit = "*"
+diff-cover = "*"
+
+[tool.coverage.run]
+relative_files = true
+source = ["."]