anonymization-lib 1.4.0__py3-none-any.whl

anonymization_lib/__init__.py

@@ -0,0 +1,7 @@
+from .metrics import KAnonymity, LDiversity, TCloseness
+from .techniques import Substitution, Suppression, Generalization
+from .visualization import Visualization, AnonymizationAdvisor
+from .io import DataExporter, DataImporter
+
+__all__ = ["KAnonymity", "LDiversity", "TCloseness", "Substitution", "Suppression", "Generalization", "Visualization", "AnonymizationAdvisor", "DataExporter", "DataImporter"]
+

anonymization_lib/io/__init__.py

@@ -0,0 +1,4 @@
+from .export import DataExporter
+from .data_import import DataImporter
+
+__all__ = ["DataExporter", "DataImporter"]

anonymization_lib/io/data_import.py

@@ -0,0 +1,90 @@
+import os
+from pyspark.sql import DataFrame, SparkSession
+
+
+class DataImporter:
+    """
+    Utility class for importing datasets into Spark DataFrames.
+
+    Supported formats:
+    - csv
+    - parquet
+    - orc
+    - s3
+    """
+
+    VALID_FORMATS = {"csv", "parquet", "orc", "s3"}
+
+    @staticmethod
+    def import_data(
+        spark: SparkSession,
+        path: str,
+        file_format: str = "parquet",
+        header: bool = True,
+        infer_schema: bool = True
+    ) -> DataFrame:
+        """
+        Imports a dataset into a Spark DataFrame.
+
+        Parameters
+        ----------
+        spark : pyspark.sql.SparkSession
+            Active Spark session.
+        path : str
+            Source dataset path.
+        file_format : str, default="parquet"
+            Input format. Supported values are:
+            - 'csv'
+            - 'parquet'
+            - 'orc'
+            - 's3'
+        header : bool, default=True
+            Whether the CSV file contains a header row.
+            Ignored for non-CSV formats.
+        infer_schema : bool, default=True
+            Whether Spark should infer the schema automatically for CSV files.
+            Ignored for non-CSV formats.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            Imported Spark DataFrame.
+
+        Raises
+        ------
+        TypeError
+            If spark is not a SparkSession.
+        ValueError
+            If path or file_format are invalid.
+        FileNotFoundError
+            If the local source path does not exist.
+        """
+        if not hasattr(spark, "read"):
+            raise TypeError("spark must provide a Spark-compatible read interface.")
+
+        if not isinstance(path, str) or not path.strip():
+            raise ValueError("The source path must be a non-empty string.")
+
+        if file_format not in DataImporter.VALID_FORMATS:
+            raise ValueError(
+                f"Unsupported format. Use one of: {DataImporter.VALID_FORMATS}"
+            )
+
+        if file_format == "s3":
+            if not path.startswith("s3a://"):
+                raise ValueError("S3 paths must start with 's3a://'")
+            return spark.read.parquet(path)
+
+        if file_format == "csv":
+            return (
+                spark.read
+                .option("header", header)
+                .option("inferSchema", infer_schema)
+                .csv(path)
+            )
+
+        if file_format == "parquet":
+            return spark.read.parquet(path)
+
+        if file_format == "orc":
+            return spark.read.orc(path)

anonymization_lib/io/export.py

@@ -0,0 +1,87 @@
+from pyspark.sql import DataFrame
+import os
+
+
+class DataExporter:
+    """
+    Utility class for exporting Spark DataFrames to different storage formats.
+
+    Supported formats:
+    - csv
+    - parquet
+    - orc
+    """
+
+    VALID_FORMATS = {"csv", "parquet", "orc"}
+    VALID_MODES = {"overwrite", "append", "ignore", "error", "errorifexists"}
+
+    @staticmethod
+    def export(
+        df: DataFrame,
+        path: str,
+        file_format: str = "parquet",
+        mode: str = "overwrite",
+        header: bool = True
+    ) -> str:
+        """
+        Exports a Spark DataFrame to the specified storage format.
+
+        Parameters
+        ----------
+        df : pyspark.sql.DataFrame
+            Spark DataFrame to export.
+        path : str
+            Destination path where the dataset will be written.
+        file_format : str, default="parquet"
+            Output format. Supported values are:
+            - 'csv'
+            - 'parquet'
+            - 'orc'
+        mode : str, default="overwrite"
+            Spark write mode. Supported values are:
+            - 'overwrite'
+            - 'append'
+            - 'ignore'
+            - 'error'
+            - 'errorifexists'
+        header : bool, default=True
+            Whether to include a header when exporting to CSV.
+            Ignored for Parquet and ORC.
+
+        Returns
+        -------
+        str
+            Confirmation message indicating that the export was completed.
+        """
+        if not hasattr(df, "write"):
+            raise TypeError("df must provide a Spark-compatible write interface.")
+
+        if file_format not in DataExporter.VALID_FORMATS:
+            raise ValueError(
+                f"Unsupported format. Use one of: {DataExporter.VALID_FORMATS}"
+            )
+
+        if mode not in DataExporter.VALID_MODES:
+            raise ValueError(
+                f"Unsupported mode. Use one of: {DataExporter.VALID_MODES}"
+            )
+
+        if not isinstance(path, str) or not path.strip():
+            raise ValueError("The destination path must be a non-empty string.")
+
+        writer = df.write.mode(mode)
+
+        if file_format == "csv":
+            writer.option("header", header).csv(path)
+
+        elif file_format == "parquet":
+            writer.parquet(path)
+
+        elif file_format == "orc":
+            writer.orc(path)
+
+        return (
+            f"Dataset successfully exported in '{file_format}' format to: "
+            f"{path}"
+        )
+

anonymization_lib/metrics/__init__.py

@@ -0,0 +1,5 @@
+from .k_anonymity import KAnonymity
+from .l_diversity import LDiversity
+from .t_closeness import TCloseness
+
+__all__ = ["KAnonymity", "LDiversity", "TCloseness"]

anonymization_lib/metrics/k_anonymity.py

@@ -0,0 +1,391 @@
+from pyspark.sql import DataFrame
+from pyspark.sql import functions as F
+
+from ..utils import EquivalenceGroups
+
+
+class KAnonymity(EquivalenceGroups):
+    """
+    Calculates k-anonymity for a dataset based on a set of
+    quasi-identifiers.
+
+    K-anonymity measures whether each combination of quasi-identifier
+    values is shared by at least ``k`` records.
+
+    Records with the same values for all quasi-identifiers form an
+    equivalence group. The size of the smallest equivalence group
+    represents the k-anonymity level of the dataset.
+
+    Parameters
+    ----------
+    quasi_identifiers : list of str, optional
+        Columns used as quasi-identifiers to construct equivalence groups.
+
+    k_threshold : int, default=2
+        Minimum required size for every equivalence group.
+
+        A dataset satisfies the configured k-anonymity requirement when
+        every equivalence group contains at least ``k_threshold`` records.
+
+    Notes
+    -----
+    Equivalence groups are computed using native Spark operations, allowing
+    the calculation to be performed in a distributed manner.
+
+    No dataset records are collected to the driver.
+
+    The computation is lazy. Spark execution occurs only when an action is
+    performed on one of the resulting DataFrames.
+    """
+
+    def __init__(
+        self,
+        quasi_identifiers=None,
+        k_threshold: int = 2
+    ):
+        """
+        Initializes the k-anonymity metric.
+
+        Parameters
+        ----------
+        quasi_identifiers : list of str, optional
+            Columns used as quasi-identifiers.
+
+            Column names must be non-empty strings and duplicate columns
+            are not allowed.
+
+        k_threshold : int, default=2
+            Minimum required size of every equivalence group.
+
+            The value must be an integer greater than 1.
+
+        Raises
+        ------
+        ValueError
+            If ``quasi_identifiers`` is not a list, contains invalid or
+            duplicate column names, or ``k_threshold`` is not an integer
+            greater than 1.
+        """
+        if quasi_identifiers is not None:
+            if not isinstance(quasi_identifiers, list):
+                raise ValueError(
+                    "'quasi_identifiers' must be a list of column names."
+                )
+
+            if not all(
+                isinstance(column, str) and column.strip()
+                for column in quasi_identifiers
+            ):
+                raise ValueError(
+                    "'quasi_identifiers' must contain only "
+                    "non-empty strings."
+                )
+
+            if len(quasi_identifiers) != len(set(quasi_identifiers)):
+                raise ValueError(
+                    "'quasi_identifiers' must not contain "
+                    "duplicate columns."
+                )
+
+        if (
+            not isinstance(k_threshold, int)
+            or isinstance(k_threshold, bool)
+            or k_threshold <= 1
+        ):
+            raise ValueError(
+                "'k_threshold' must be an integer greater than 1."
+            )
+
+        self.quasi_identifiers = quasi_identifiers or []
+        self.k_threshold = k_threshold
+
+    def summary(
+        self,
+        df: DataFrame
+    ):
+        """
+        Computes k-anonymity statistics for the input dataset.
+
+        Records are grouped according to their values for the configured
+        quasi-identifiers. Each resulting equivalence group contains a
+        ``group_size`` representing the number of records sharing that
+        combination of quasi-identifier values.
+
+        The k-anonymity value of the dataset is defined as the size of the
+        smallest equivalence group:
+
+            k = min(group_size)
+
+        Equivalence groups whose size is smaller than ``k_threshold`` are
+        considered violating groups.
+
+        Parameters
+        ----------
+        df : pyspark.sql.DataFrame
+            Input dataset containing the configured quasi-identifiers.
+
+        Returns
+        -------
+        KAnonymityResult
+            Container containing:
+
+            - ``summary_df``:
+                DataFrame with aggregated k-anonymity statistics.
+
+            - ``violating_groups``:
+                Equivalence groups whose size is below the configured
+                ``k_threshold``.
+
+        Raises
+        ------
+        ValueError
+            If the input DataFrame is None, is not Spark-compatible, or
+            does not contain one or more configured quasi-identifiers.
+
+        Notes
+        -----
+        The computation uses native Spark aggregations and is performed
+        in a distributed manner.
+
+        This method does not trigger Spark execution by itself. Both
+        returned DataFrames are lazy and are evaluated only when a Spark
+        action is performed.
+
+        ``summary_df`` and ``violating_groups`` share the same logical
+        equivalence-group computation. Without persistence, separate
+        actions on both result DataFrames may cause Spark to recompute
+        that shared lineage.
+        """
+        if df is None:
+            raise ValueError(
+                "Input DataFrame cannot be None."
+            )
+
+        # Compatible with standard Spark and Spark Connect.
+        if not hasattr(df, "groupBy"):
+            raise ValueError(
+                "Input must be a Spark-compatible DataFrame."
+            )
+
+        self._validate_columns(
+            df,
+            self.quasi_identifiers
+        )
+
+        # ------------------------------------------------------
+        # EQUIVALENCE GROUPS
+        # ------------------------------------------------------
+
+        equivalence_groups = self._build_equivalence_groups(
+            df,
+            self.quasi_identifiers
+        )
+
+        # ------------------------------------------------------
+        # VIOLATING GROUPS
+        # ------------------------------------------------------
+
+        violating_groups = equivalence_groups.filter(
+            F.col("group_size") < self.k_threshold
+        )
+
+        # ------------------------------------------------------
+        # SUMMARY
+        # ------------------------------------------------------
+
+        summary_df = (
+            equivalence_groups
+            .agg(
+                F.count("*").alias(
+                    "num_equivalence_groups"
+                ),
+                F.min("group_size").alias(
+                    "k_value"
+                ),
+                F.max("group_size").alias(
+                    "max_group_size"
+                ),
+                F.round(
+                    F.avg("group_size"),
+                    2
+                ).alias(
+                    "avg_group_size"
+                ),
+                F.sum(
+                    F.when(
+                        F.col("group_size") < self.k_threshold,
+                        1
+                    ).otherwise(0)
+                ).alias(
+                    "num_violating_groups"
+                )
+            )
+            .withColumn(
+                "k_threshold",
+                F.lit(self.k_threshold)
+            )
+            .withColumn(
+                "satisfies_k_anonymity",
+                F.col("k_value")
+                >= F.col("k_threshold")
+            )
+        )
+
+        return KAnonymityResult(
+            summary_df=summary_df,
+            violating_groups=violating_groups
+        )
+
+
+class KAnonymityResult:
+    """
+    Container for k-anonymity results.
+
+    The object stores both the aggregated summary of the k-anonymity
+    analysis and the equivalence groups that violate the configured
+    k-threshold.
+
+    Parameters
+    ----------
+    summary_df : pyspark.sql.DataFrame
+        DataFrame containing aggregated k-anonymity statistics.
+
+    violating_groups : pyspark.sql.DataFrame
+        Equivalence groups whose size is below the configured k-threshold.
+
+    Notes
+    -----
+    The stored DataFrames remain lazy Spark DataFrames. Creating this
+    result object does not trigger Spark execution.
+    """
+
+    def __init__(
+        self,
+        summary_df: DataFrame,
+        violating_groups: DataFrame
+    ):
+        """
+        Initializes the k-anonymity result container.
+
+        Parameters
+        ----------
+        summary_df : pyspark.sql.DataFrame
+            Aggregated statistics of k-anonymity across equivalence groups.
+
+        violating_groups : pyspark.sql.DataFrame
+            Subset of equivalence groups whose size is below the configured
+            k-threshold, representing potential privacy risks.
+        """
+        self.summary_df = summary_df
+        self.violating_groups = violating_groups
+
+    def show_summary(self) -> None:
+        """
+        Displays the aggregated k-anonymity summary.
+
+        Returns
+        -------
+        None
+            The summary is displayed using Spark's ``show`` method.
+
+        Notes
+        -----
+        Calling this method triggers Spark execution.
+        """
+        self.summary_df.show(
+            truncate=False
+        )
+
+    def show_violating_groups(
+        self,
+        n: int = 10,
+        sort: bool = False
+    ) -> None:
+        """
+        Displays equivalence groups that do not satisfy k-anonymity.
+
+        Parameters
+        ----------
+        n : int, default=10
+            Maximum number of violating groups to display.
+
+        sort : bool, default=False
+            Whether to sort violating groups by ``group_size`` in
+            ascending order before displaying them.
+
+            Sorting may increase computational cost for large datasets
+            because it can require an additional distributed ordering
+            operation.
+
+        Returns
+        -------
+        None
+            The selected violating groups are displayed using Spark's
+            ``show`` method.
+
+        Raises
+        ------
+        ValueError
+            If ``n`` is not a positive integer or ``sort`` is not a
+            boolean.
+
+        Notes
+        -----
+        Calling this method triggers Spark execution.
+        """
+        if (
+            not isinstance(n, int)
+            or isinstance(n, bool)
+            or n <= 0
+        ):
+            raise ValueError(
+                "'n' must be a positive integer."
+            )
+
+        if not isinstance(sort, bool):
+            raise ValueError(
+                "'sort' must be a boolean."
+            )
+
+        df = self.violating_groups
+
+        if sort:
+            df = df.orderBy(
+                F.col("group_size").asc()
+            )
+
+        df.show(
+            n,
+            truncate=False
+        )
+
+    def get_summary_df(self) -> DataFrame:
+        """
+        Returns the aggregated k-anonymity summary.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            DataFrame containing the aggregated k-anonymity statistics.
+
+        Notes
+        -----
+        Returning the DataFrame does not trigger Spark execution.
+        """
+        return self.summary_df
+
+    def get_violating_groups(self) -> DataFrame:
+        """
+        Returns the equivalence groups that violate k-anonymity.
+
+        Returns
+        -------
+        pyspark.sql.DataFrame
+            Equivalence groups whose size is below the configured
+            k-threshold.
+
+        Notes
+        -----
+        Returning the DataFrame does not trigger Spark execution.
+        """
+        return self.violating_groups