anonymization-lib 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anonymization_lib/__init__.py +7 -0
- anonymization_lib/io/__init__.py +4 -0
- anonymization_lib/io/data_import.py +90 -0
- anonymization_lib/io/export.py +87 -0
- anonymization_lib/metrics/__init__.py +5 -0
- anonymization_lib/metrics/k_anonymity.py +391 -0
- anonymization_lib/metrics/l_diversity.py +393 -0
- anonymization_lib/metrics/t_closeness.py +618 -0
- anonymization_lib/techniques/__init__.py +6 -0
- anonymization_lib/techniques/generalization.py +907 -0
- anonymization_lib/techniques/pipeline.py +134 -0
- anonymization_lib/techniques/substitution.py +343 -0
- anonymization_lib/techniques/suppression.py +255 -0
- anonymization_lib/utils.py +153 -0
- anonymization_lib/visualization/Suggestion.py +730 -0
- anonymization_lib/visualization/__init__.py +4 -0
- anonymization_lib/visualization/visualization.py +845 -0
- anonymization_lib-1.4.0.dist-info/METADATA +302 -0
- anonymization_lib-1.4.0.dist-info/RECORD +22 -0
- anonymization_lib-1.4.0.dist-info/WHEEL +5 -0
- anonymization_lib-1.4.0.dist-info/licenses/LICENSE +21 -0
- anonymization_lib-1.4.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
from pyspark.sql import DataFrame
|
|
2
|
+
from pyspark.sql import functions as F
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Suppression:
|
|
6
|
+
"""
|
|
7
|
+
Applies suppression techniques to one or multiple columns.
|
|
8
|
+
|
|
9
|
+
Suppression removes or hides information from selected attributes in
|
|
10
|
+
order to reduce the amount of identifying information available in
|
|
11
|
+
the dataset.
|
|
12
|
+
|
|
13
|
+
Two suppression modes are supported:
|
|
14
|
+
|
|
15
|
+
- Null suppression:
|
|
16
|
+
Replaces every value in the selected column with NULL while
|
|
17
|
+
preserving the original Spark data type.
|
|
18
|
+
|
|
19
|
+
- Drop suppression:
|
|
20
|
+
Removes the selected column completely from the dataset.
|
|
21
|
+
|
|
22
|
+
Parameters
|
|
23
|
+
----------
|
|
24
|
+
columns_modes : dict
|
|
25
|
+
Dictionary where keys are column names and values specify the
|
|
26
|
+
suppression mode to apply.
|
|
27
|
+
|
|
28
|
+
Supported modes are:
|
|
29
|
+
|
|
30
|
+
- ``"null"``:
|
|
31
|
+
Replaces all values in the column with NULL.
|
|
32
|
+
|
|
33
|
+
- ``"drop"``:
|
|
34
|
+
Removes the column from the dataset.
|
|
35
|
+
|
|
36
|
+
Example::
|
|
37
|
+
|
|
38
|
+
{
|
|
39
|
+
"age": "null",
|
|
40
|
+
"name": "drop"
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
Notes
|
|
44
|
+
-----
|
|
45
|
+
The transformation is implemented entirely using native Spark
|
|
46
|
+
DataFrame operations.
|
|
47
|
+
|
|
48
|
+
No records are collected to the driver and no Python UDF is used,
|
|
49
|
+
allowing suppression to be executed in a distributed manner.
|
|
50
|
+
|
|
51
|
+
Columns suppressed using ``"null"`` preserve their original Spark
|
|
52
|
+
data type.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
VALID_MODES = {
|
|
56
|
+
"null",
|
|
57
|
+
"drop"
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
def __init__(
|
|
61
|
+
self,
|
|
62
|
+
columns_modes: dict
|
|
63
|
+
):
|
|
64
|
+
"""
|
|
65
|
+
Initializes the suppression transformation.
|
|
66
|
+
|
|
67
|
+
Parameters
|
|
68
|
+
----------
|
|
69
|
+
columns_modes : dict
|
|
70
|
+
Dictionary where keys are column names and values are
|
|
71
|
+
suppression modes.
|
|
72
|
+
|
|
73
|
+
Supported modes are:
|
|
74
|
+
|
|
75
|
+
- ``"null"``:
|
|
76
|
+
Replaces all values in the column with NULL while
|
|
77
|
+
preserving the original column data type.
|
|
78
|
+
|
|
79
|
+
- ``"drop"``:
|
|
80
|
+
Removes the column from the dataset.
|
|
81
|
+
|
|
82
|
+
Example::
|
|
83
|
+
|
|
84
|
+
{
|
|
85
|
+
"age": "null",
|
|
86
|
+
"name": "drop"
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
Raises
|
|
90
|
+
------
|
|
91
|
+
ValueError
|
|
92
|
+
If ``columns_modes`` is not a dictionary, is empty, contains
|
|
93
|
+
invalid or empty column names, or contains unsupported
|
|
94
|
+
suppression modes.
|
|
95
|
+
"""
|
|
96
|
+
if not isinstance(columns_modes, dict):
|
|
97
|
+
raise ValueError(
|
|
98
|
+
"'columns_modes' must be a dictionary."
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
if not columns_modes:
|
|
102
|
+
raise ValueError(
|
|
103
|
+
"'columns_modes' must not be empty."
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
if not all(
|
|
107
|
+
isinstance(column, str) and column.strip()
|
|
108
|
+
for column in columns_modes
|
|
109
|
+
):
|
|
110
|
+
raise ValueError(
|
|
111
|
+
"All column names must be non-empty strings."
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
if not all(
|
|
115
|
+
isinstance(mode, str)
|
|
116
|
+
for mode in columns_modes.values()
|
|
117
|
+
):
|
|
118
|
+
raise ValueError(
|
|
119
|
+
"All suppression modes must be strings."
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
for column, mode in columns_modes.items():
|
|
123
|
+
if mode not in self.VALID_MODES:
|
|
124
|
+
raise ValueError(
|
|
125
|
+
f"Invalid mode '{mode}' for column '{column}'. "
|
|
126
|
+
f"Use {sorted(self.VALID_MODES)}."
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
self.columns_modes = columns_modes.copy()
|
|
130
|
+
|
|
131
|
+
def transform(
|
|
132
|
+
self,
|
|
133
|
+
df: DataFrame
|
|
134
|
+
) -> DataFrame:
|
|
135
|
+
"""
|
|
136
|
+
Applies the configured suppression operations to the input dataset.
|
|
137
|
+
|
|
138
|
+
Columns configured with ``"drop"`` are removed from the resulting
|
|
139
|
+
DataFrame.
|
|
140
|
+
|
|
141
|
+
Columns configured with ``"null"`` are preserved but every value
|
|
142
|
+
is replaced with NULL. The original Spark data type of each column
|
|
143
|
+
is maintained.
|
|
144
|
+
|
|
145
|
+
Parameters
|
|
146
|
+
----------
|
|
147
|
+
df : pyspark.sql.DataFrame
|
|
148
|
+
Input dataset containing the columns to suppress.
|
|
149
|
+
|
|
150
|
+
Returns
|
|
151
|
+
-------
|
|
152
|
+
pyspark.sql.DataFrame
|
|
153
|
+
New DataFrame with the configured suppression operations
|
|
154
|
+
applied.
|
|
155
|
+
|
|
156
|
+
Raises
|
|
157
|
+
------
|
|
158
|
+
ValueError
|
|
159
|
+
If the input DataFrame is None, is not Spark-compatible, or
|
|
160
|
+
does not contain one or more of the configured columns.
|
|
161
|
+
|
|
162
|
+
Notes
|
|
163
|
+
-----
|
|
164
|
+
All configured columns are validated before any transformation is
|
|
165
|
+
constructed.
|
|
166
|
+
|
|
167
|
+
Drop suppression is applied to all selected columns in a single
|
|
168
|
+
Spark ``drop`` operation.
|
|
169
|
+
|
|
170
|
+
Null suppression is applied through a single projection, avoiding
|
|
171
|
+
the construction of multiple consecutive ``withColumn``
|
|
172
|
+
transformations.
|
|
173
|
+
|
|
174
|
+
The transformation is lazy and uses only native Spark expressions.
|
|
175
|
+
Dataset records are not collected to the driver.
|
|
176
|
+
"""
|
|
177
|
+
if df is None:
|
|
178
|
+
raise ValueError(
|
|
179
|
+
"Input DataFrame cannot be None."
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
# Compatible with standard Spark and Spark Connect.
|
|
183
|
+
if not hasattr(df, "select"):
|
|
184
|
+
raise ValueError(
|
|
185
|
+
"Input must be a Spark-compatible DataFrame."
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# ------------------------------------------------------
|
|
189
|
+
# VALIDATE COLUMNS
|
|
190
|
+
# ------------------------------------------------------
|
|
191
|
+
|
|
192
|
+
missing_columns = [
|
|
193
|
+
column
|
|
194
|
+
for column in self.columns_modes
|
|
195
|
+
if column not in df.columns
|
|
196
|
+
]
|
|
197
|
+
|
|
198
|
+
if missing_columns:
|
|
199
|
+
raise ValueError(
|
|
200
|
+
"Columns not found in DataFrame: "
|
|
201
|
+
+ ", ".join(missing_columns)
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
# ------------------------------------------------------
|
|
205
|
+
# SEPARATE SUPPRESSION MODES
|
|
206
|
+
# ------------------------------------------------------
|
|
207
|
+
|
|
208
|
+
drop_columns = [
|
|
209
|
+
column
|
|
210
|
+
for column, mode in self.columns_modes.items()
|
|
211
|
+
if mode == "drop"
|
|
212
|
+
]
|
|
213
|
+
|
|
214
|
+
null_columns = {
|
|
215
|
+
column
|
|
216
|
+
for column, mode in self.columns_modes.items()
|
|
217
|
+
if mode == "null"
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
# ------------------------------------------------------
|
|
221
|
+
# NULL SUPPRESSION
|
|
222
|
+
# ------------------------------------------------------
|
|
223
|
+
|
|
224
|
+
if null_columns:
|
|
225
|
+
schema_types = {
|
|
226
|
+
field.name: field.dataType
|
|
227
|
+
for field in df.schema.fields
|
|
228
|
+
if field.name in null_columns
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
result_df = df.select(
|
|
232
|
+
*[
|
|
233
|
+
(
|
|
234
|
+
F.lit(None)
|
|
235
|
+
.cast(schema_types[column])
|
|
236
|
+
.alias(column)
|
|
237
|
+
if column in null_columns
|
|
238
|
+
else F.col(column)
|
|
239
|
+
)
|
|
240
|
+
for column in df.columns
|
|
241
|
+
]
|
|
242
|
+
)
|
|
243
|
+
else:
|
|
244
|
+
result_df = df
|
|
245
|
+
|
|
246
|
+
# ------------------------------------------------------
|
|
247
|
+
# DROP SUPPRESSION
|
|
248
|
+
# ------------------------------------------------------
|
|
249
|
+
|
|
250
|
+
if drop_columns:
|
|
251
|
+
result_df = result_df.drop(
|
|
252
|
+
*drop_columns
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
return result_df
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
from pyspark.sql import DataFrame
|
|
2
|
+
from pyspark.sql import functions as F
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def get_dtype(df: DataFrame, column: str) -> str:
|
|
6
|
+
"""
|
|
7
|
+
Returns the Spark data type of a column.
|
|
8
|
+
|
|
9
|
+
Parameters
|
|
10
|
+
----------
|
|
11
|
+
df : pyspark.sql.DataFrame
|
|
12
|
+
column : str
|
|
13
|
+
|
|
14
|
+
Returns
|
|
15
|
+
-------
|
|
16
|
+
str
|
|
17
|
+
Data type of the column.
|
|
18
|
+
"""
|
|
19
|
+
if column not in df.columns:
|
|
20
|
+
raise ValueError(f"Column '{column}' does not exist in DataFrame.")
|
|
21
|
+
|
|
22
|
+
return dict(df.dtypes)[column]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def is_numeric(dtype: str) -> bool:
|
|
26
|
+
"""
|
|
27
|
+
Checks if a dtype corresponds to a numeric type.
|
|
28
|
+
"""
|
|
29
|
+
return (
|
|
30
|
+
dtype in [
|
|
31
|
+
"int", "bigint", "double", "float",
|
|
32
|
+
"long", "smallint", "tinyint"
|
|
33
|
+
]
|
|
34
|
+
or dtype.startswith("decimal")
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def is_date(dtype: str) -> bool:
|
|
39
|
+
"""
|
|
40
|
+
Checks if a dtype corresponds to a date or timestamp.
|
|
41
|
+
"""
|
|
42
|
+
return dtype in ["date", "timestamp"]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def is_boolean(dtype: str) -> bool:
|
|
46
|
+
"""
|
|
47
|
+
Checks if a dtype corresponds to a boolean type.
|
|
48
|
+
"""
|
|
49
|
+
return dtype == "boolean"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def infer_semantic_type(df: DataFrame, column: str) -> str:
|
|
53
|
+
"""
|
|
54
|
+
Infers the semantic type of a column.
|
|
55
|
+
|
|
56
|
+
Parameters
|
|
57
|
+
----------
|
|
58
|
+
df : pyspark.sql.DataFrame
|
|
59
|
+
column : str
|
|
60
|
+
|
|
61
|
+
Returns
|
|
62
|
+
-------
|
|
63
|
+
str
|
|
64
|
+
One of: 'numeric', 'date', 'categorical'
|
|
65
|
+
"""
|
|
66
|
+
dtype = get_dtype(df, column)
|
|
67
|
+
|
|
68
|
+
if is_numeric(dtype):
|
|
69
|
+
return "numeric"
|
|
70
|
+
|
|
71
|
+
if is_date(dtype):
|
|
72
|
+
return "date"
|
|
73
|
+
|
|
74
|
+
if is_boolean(dtype):
|
|
75
|
+
return "categorical"
|
|
76
|
+
|
|
77
|
+
return "categorical"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class EquivalenceGroups:
|
|
83
|
+
"""
|
|
84
|
+
Base class providing common utilities for building and validating
|
|
85
|
+
equivalence groups based on quasi-identifiers in a dataset.
|
|
86
|
+
"""
|
|
87
|
+
def _validate_columns(self, df: DataFrame, columns: list):
|
|
88
|
+
"""
|
|
89
|
+
Validates that the provided columns exist in the DataFrame.
|
|
90
|
+
|
|
91
|
+
Parameters
|
|
92
|
+
----------
|
|
93
|
+
df : pyspark.sql.DataFrame
|
|
94
|
+
Input dataset.
|
|
95
|
+
columns : list of str
|
|
96
|
+
List of column names to validate.
|
|
97
|
+
|
|
98
|
+
Raises
|
|
99
|
+
------
|
|
100
|
+
ValueError
|
|
101
|
+
If the list of columns is empty or if any column does not exist
|
|
102
|
+
in the DataFrame.
|
|
103
|
+
"""
|
|
104
|
+
if not columns:
|
|
105
|
+
raise ValueError("At least one column must be provided.")
|
|
106
|
+
|
|
107
|
+
missing_cols = [c for c in columns if c not in df.columns]
|
|
108
|
+
if missing_cols:
|
|
109
|
+
raise ValueError(f"The following columns do not exist in the DataFrame: {missing_cols}")
|
|
110
|
+
|
|
111
|
+
def _build_equivalence_groups(self,df: DataFrame,quasi_identifiers: list,extra_aggs: list = None) -> DataFrame:
|
|
112
|
+
"""
|
|
113
|
+
Builds equivalence groups based on the provided quasi-identifiers.
|
|
114
|
+
|
|
115
|
+
Each group represents a unique combination of quasi-identifier values,
|
|
116
|
+
and includes the size of the group. Additional aggregations can be
|
|
117
|
+
optionally applied (e.g., diversity of a sensitive attribute).
|
|
118
|
+
|
|
119
|
+
Parameters
|
|
120
|
+
----------
|
|
121
|
+
df : pyspark.sql.DataFrame
|
|
122
|
+
Input dataset.
|
|
123
|
+
quasi_identifiers : list of str
|
|
124
|
+
Columns used to define equivalence groups.
|
|
125
|
+
extra_aggs : list of pyspark.sql.Column, optional
|
|
126
|
+
Additional aggregation expressions to apply within each group.
|
|
127
|
+
For example:
|
|
128
|
+
- F.countDistinct("sensitive_col").alias("l_diversity")
|
|
129
|
+
|
|
130
|
+
Returns
|
|
131
|
+
-------
|
|
132
|
+
pyspark.sql.DataFrame
|
|
133
|
+
DataFrame containing one row per equivalence group, with:
|
|
134
|
+
- quasi-identifier columns
|
|
135
|
+
- 'group_size' (number of records in the group)
|
|
136
|
+
- any additional aggregation columns provided
|
|
137
|
+
|
|
138
|
+
Notes
|
|
139
|
+
-----
|
|
140
|
+
This method is used as a common building block for privacy metrics
|
|
141
|
+
such as k-anonymity, l-diversity and t-closeness.
|
|
142
|
+
"""
|
|
143
|
+
aggs = [F.count("*").alias("group_size")]
|
|
144
|
+
|
|
145
|
+
if extra_aggs is not None:
|
|
146
|
+
aggs.extend(extra_aggs)
|
|
147
|
+
|
|
148
|
+
return df.groupBy(*quasi_identifiers).agg(*aggs)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
|