anonymization-lib 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,255 @@
1
+ from pyspark.sql import DataFrame
2
+ from pyspark.sql import functions as F
3
+
4
+
5
+ class Suppression:
6
+ """
7
+ Applies suppression techniques to one or multiple columns.
8
+
9
+ Suppression removes or hides information from selected attributes in
10
+ order to reduce the amount of identifying information available in
11
+ the dataset.
12
+
13
+ Two suppression modes are supported:
14
+
15
+ - Null suppression:
16
+ Replaces every value in the selected column with NULL while
17
+ preserving the original Spark data type.
18
+
19
+ - Drop suppression:
20
+ Removes the selected column completely from the dataset.
21
+
22
+ Parameters
23
+ ----------
24
+ columns_modes : dict
25
+ Dictionary where keys are column names and values specify the
26
+ suppression mode to apply.
27
+
28
+ Supported modes are:
29
+
30
+ - ``"null"``:
31
+ Replaces all values in the column with NULL.
32
+
33
+ - ``"drop"``:
34
+ Removes the column from the dataset.
35
+
36
+ Example::
37
+
38
+ {
39
+ "age": "null",
40
+ "name": "drop"
41
+ }
42
+
43
+ Notes
44
+ -----
45
+ The transformation is implemented entirely using native Spark
46
+ DataFrame operations.
47
+
48
+ No records are collected to the driver and no Python UDF is used,
49
+ allowing suppression to be executed in a distributed manner.
50
+
51
+ Columns suppressed using ``"null"`` preserve their original Spark
52
+ data type.
53
+ """
54
+
55
+ VALID_MODES = {
56
+ "null",
57
+ "drop"
58
+ }
59
+
60
+ def __init__(
61
+ self,
62
+ columns_modes: dict
63
+ ):
64
+ """
65
+ Initializes the suppression transformation.
66
+
67
+ Parameters
68
+ ----------
69
+ columns_modes : dict
70
+ Dictionary where keys are column names and values are
71
+ suppression modes.
72
+
73
+ Supported modes are:
74
+
75
+ - ``"null"``:
76
+ Replaces all values in the column with NULL while
77
+ preserving the original column data type.
78
+
79
+ - ``"drop"``:
80
+ Removes the column from the dataset.
81
+
82
+ Example::
83
+
84
+ {
85
+ "age": "null",
86
+ "name": "drop"
87
+ }
88
+
89
+ Raises
90
+ ------
91
+ ValueError
92
+ If ``columns_modes`` is not a dictionary, is empty, contains
93
+ invalid or empty column names, or contains unsupported
94
+ suppression modes.
95
+ """
96
+ if not isinstance(columns_modes, dict):
97
+ raise ValueError(
98
+ "'columns_modes' must be a dictionary."
99
+ )
100
+
101
+ if not columns_modes:
102
+ raise ValueError(
103
+ "'columns_modes' must not be empty."
104
+ )
105
+
106
+ if not all(
107
+ isinstance(column, str) and column.strip()
108
+ for column in columns_modes
109
+ ):
110
+ raise ValueError(
111
+ "All column names must be non-empty strings."
112
+ )
113
+
114
+ if not all(
115
+ isinstance(mode, str)
116
+ for mode in columns_modes.values()
117
+ ):
118
+ raise ValueError(
119
+ "All suppression modes must be strings."
120
+ )
121
+
122
+ for column, mode in columns_modes.items():
123
+ if mode not in self.VALID_MODES:
124
+ raise ValueError(
125
+ f"Invalid mode '{mode}' for column '{column}'. "
126
+ f"Use {sorted(self.VALID_MODES)}."
127
+ )
128
+
129
+ self.columns_modes = columns_modes.copy()
130
+
131
+ def transform(
132
+ self,
133
+ df: DataFrame
134
+ ) -> DataFrame:
135
+ """
136
+ Applies the configured suppression operations to the input dataset.
137
+
138
+ Columns configured with ``"drop"`` are removed from the resulting
139
+ DataFrame.
140
+
141
+ Columns configured with ``"null"`` are preserved but every value
142
+ is replaced with NULL. The original Spark data type of each column
143
+ is maintained.
144
+
145
+ Parameters
146
+ ----------
147
+ df : pyspark.sql.DataFrame
148
+ Input dataset containing the columns to suppress.
149
+
150
+ Returns
151
+ -------
152
+ pyspark.sql.DataFrame
153
+ New DataFrame with the configured suppression operations
154
+ applied.
155
+
156
+ Raises
157
+ ------
158
+ ValueError
159
+ If the input DataFrame is None, is not Spark-compatible, or
160
+ does not contain one or more of the configured columns.
161
+
162
+ Notes
163
+ -----
164
+ All configured columns are validated before any transformation is
165
+ constructed.
166
+
167
+ Drop suppression is applied to all selected columns in a single
168
+ Spark ``drop`` operation.
169
+
170
+ Null suppression is applied through a single projection, avoiding
171
+ the construction of multiple consecutive ``withColumn``
172
+ transformations.
173
+
174
+ The transformation is lazy and uses only native Spark expressions.
175
+ Dataset records are not collected to the driver.
176
+ """
177
+ if df is None:
178
+ raise ValueError(
179
+ "Input DataFrame cannot be None."
180
+ )
181
+
182
+ # Compatible with standard Spark and Spark Connect.
183
+ if not hasattr(df, "select"):
184
+ raise ValueError(
185
+ "Input must be a Spark-compatible DataFrame."
186
+ )
187
+
188
+ # ------------------------------------------------------
189
+ # VALIDATE COLUMNS
190
+ # ------------------------------------------------------
191
+
192
+ missing_columns = [
193
+ column
194
+ for column in self.columns_modes
195
+ if column not in df.columns
196
+ ]
197
+
198
+ if missing_columns:
199
+ raise ValueError(
200
+ "Columns not found in DataFrame: "
201
+ + ", ".join(missing_columns)
202
+ )
203
+
204
+ # ------------------------------------------------------
205
+ # SEPARATE SUPPRESSION MODES
206
+ # ------------------------------------------------------
207
+
208
+ drop_columns = [
209
+ column
210
+ for column, mode in self.columns_modes.items()
211
+ if mode == "drop"
212
+ ]
213
+
214
+ null_columns = {
215
+ column
216
+ for column, mode in self.columns_modes.items()
217
+ if mode == "null"
218
+ }
219
+
220
+ # ------------------------------------------------------
221
+ # NULL SUPPRESSION
222
+ # ------------------------------------------------------
223
+
224
+ if null_columns:
225
+ schema_types = {
226
+ field.name: field.dataType
227
+ for field in df.schema.fields
228
+ if field.name in null_columns
229
+ }
230
+
231
+ result_df = df.select(
232
+ *[
233
+ (
234
+ F.lit(None)
235
+ .cast(schema_types[column])
236
+ .alias(column)
237
+ if column in null_columns
238
+ else F.col(column)
239
+ )
240
+ for column in df.columns
241
+ ]
242
+ )
243
+ else:
244
+ result_df = df
245
+
246
+ # ------------------------------------------------------
247
+ # DROP SUPPRESSION
248
+ # ------------------------------------------------------
249
+
250
+ if drop_columns:
251
+ result_df = result_df.drop(
252
+ *drop_columns
253
+ )
254
+
255
+ return result_df
@@ -0,0 +1,153 @@
1
+ from pyspark.sql import DataFrame
2
+ from pyspark.sql import functions as F
3
+
4
+
5
+ def get_dtype(df: DataFrame, column: str) -> str:
6
+ """
7
+ Returns the Spark data type of a column.
8
+
9
+ Parameters
10
+ ----------
11
+ df : pyspark.sql.DataFrame
12
+ column : str
13
+
14
+ Returns
15
+ -------
16
+ str
17
+ Data type of the column.
18
+ """
19
+ if column not in df.columns:
20
+ raise ValueError(f"Column '{column}' does not exist in DataFrame.")
21
+
22
+ return dict(df.dtypes)[column]
23
+
24
+
25
+ def is_numeric(dtype: str) -> bool:
26
+ """
27
+ Checks if a dtype corresponds to a numeric type.
28
+ """
29
+ return (
30
+ dtype in [
31
+ "int", "bigint", "double", "float",
32
+ "long", "smallint", "tinyint"
33
+ ]
34
+ or dtype.startswith("decimal")
35
+ )
36
+
37
+
38
+ def is_date(dtype: str) -> bool:
39
+ """
40
+ Checks if a dtype corresponds to a date or timestamp.
41
+ """
42
+ return dtype in ["date", "timestamp"]
43
+
44
+
45
+ def is_boolean(dtype: str) -> bool:
46
+ """
47
+ Checks if a dtype corresponds to a boolean type.
48
+ """
49
+ return dtype == "boolean"
50
+
51
+
52
+ def infer_semantic_type(df: DataFrame, column: str) -> str:
53
+ """
54
+ Infers the semantic type of a column.
55
+
56
+ Parameters
57
+ ----------
58
+ df : pyspark.sql.DataFrame
59
+ column : str
60
+
61
+ Returns
62
+ -------
63
+ str
64
+ One of: 'numeric', 'date', 'categorical'
65
+ """
66
+ dtype = get_dtype(df, column)
67
+
68
+ if is_numeric(dtype):
69
+ return "numeric"
70
+
71
+ if is_date(dtype):
72
+ return "date"
73
+
74
+ if is_boolean(dtype):
75
+ return "categorical"
76
+
77
+ return "categorical"
78
+
79
+
80
+
81
+
82
+ class EquivalenceGroups:
83
+ """
84
+ Base class providing common utilities for building and validating
85
+ equivalence groups based on quasi-identifiers in a dataset.
86
+ """
87
+ def _validate_columns(self, df: DataFrame, columns: list):
88
+ """
89
+ Validates that the provided columns exist in the DataFrame.
90
+
91
+ Parameters
92
+ ----------
93
+ df : pyspark.sql.DataFrame
94
+ Input dataset.
95
+ columns : list of str
96
+ List of column names to validate.
97
+
98
+ Raises
99
+ ------
100
+ ValueError
101
+ If the list of columns is empty or if any column does not exist
102
+ in the DataFrame.
103
+ """
104
+ if not columns:
105
+ raise ValueError("At least one column must be provided.")
106
+
107
+ missing_cols = [c for c in columns if c not in df.columns]
108
+ if missing_cols:
109
+ raise ValueError(f"The following columns do not exist in the DataFrame: {missing_cols}")
110
+
111
+ def _build_equivalence_groups(self,df: DataFrame,quasi_identifiers: list,extra_aggs: list = None) -> DataFrame:
112
+ """
113
+ Builds equivalence groups based on the provided quasi-identifiers.
114
+
115
+ Each group represents a unique combination of quasi-identifier values,
116
+ and includes the size of the group. Additional aggregations can be
117
+ optionally applied (e.g., diversity of a sensitive attribute).
118
+
119
+ Parameters
120
+ ----------
121
+ df : pyspark.sql.DataFrame
122
+ Input dataset.
123
+ quasi_identifiers : list of str
124
+ Columns used to define equivalence groups.
125
+ extra_aggs : list of pyspark.sql.Column, optional
126
+ Additional aggregation expressions to apply within each group.
127
+ For example:
128
+ - F.countDistinct("sensitive_col").alias("l_diversity")
129
+
130
+ Returns
131
+ -------
132
+ pyspark.sql.DataFrame
133
+ DataFrame containing one row per equivalence group, with:
134
+ - quasi-identifier columns
135
+ - 'group_size' (number of records in the group)
136
+ - any additional aggregation columns provided
137
+
138
+ Notes
139
+ -----
140
+ This method is used as a common building block for privacy metrics
141
+ such as k-anonymity, l-diversity and t-closeness.
142
+ """
143
+ aggs = [F.count("*").alias("group_size")]
144
+
145
+ if extra_aggs is not None:
146
+ aggs.extend(extra_aggs)
147
+
148
+ return df.groupBy(*quasi_identifiers).agg(*aggs)
149
+
150
+
151
+
152
+
153
+