anonymization-lib 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anonymization_lib/__init__.py +7 -0
- anonymization_lib/io/__init__.py +4 -0
- anonymization_lib/io/data_import.py +90 -0
- anonymization_lib/io/export.py +87 -0
- anonymization_lib/metrics/__init__.py +5 -0
- anonymization_lib/metrics/k_anonymity.py +391 -0
- anonymization_lib/metrics/l_diversity.py +393 -0
- anonymization_lib/metrics/t_closeness.py +618 -0
- anonymization_lib/techniques/__init__.py +6 -0
- anonymization_lib/techniques/generalization.py +907 -0
- anonymization_lib/techniques/pipeline.py +134 -0
- anonymization_lib/techniques/substitution.py +343 -0
- anonymization_lib/techniques/suppression.py +255 -0
- anonymization_lib/utils.py +153 -0
- anonymization_lib/visualization/Suggestion.py +730 -0
- anonymization_lib/visualization/__init__.py +4 -0
- anonymization_lib/visualization/visualization.py +845 -0
- anonymization_lib-1.4.0.dist-info/METADATA +302 -0
- anonymization_lib-1.4.0.dist-info/RECORD +22 -0
- anonymization_lib-1.4.0.dist-info/WHEEL +5 -0
- anonymization_lib-1.4.0.dist-info/licenses/LICENSE +21 -0
- anonymization_lib-1.4.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,845 @@
|
|
|
1
|
+
from pyspark.sql import DataFrame
|
|
2
|
+
from pyspark.sql import functions as F
|
|
3
|
+
import plotly.graph_objects as go
|
|
4
|
+
|
|
5
|
+
from ..utils import infer_semantic_type
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Visualization:
|
|
9
|
+
"""
|
|
10
|
+
Generates interactive visualizations for a given column in a Spark DataFrame.
|
|
11
|
+
|
|
12
|
+
The visualization generated depends on the semantic type inferred for the
|
|
13
|
+
selected column:
|
|
14
|
+
|
|
15
|
+
- Numeric columns:
|
|
16
|
+
Bar plot, histogram and box plot.
|
|
17
|
+
|
|
18
|
+
- Categorical columns:
|
|
19
|
+
Bar plot and treemap containing the most frequent categories.
|
|
20
|
+
|
|
21
|
+
- Date columns:
|
|
22
|
+
Year distribution, month distribution and year-month sunburst.
|
|
23
|
+
|
|
24
|
+
The class is designed to perform data-intensive operations using Spark
|
|
25
|
+
before transferring aggregated results to the driver for visualization
|
|
26
|
+
with Plotly.
|
|
27
|
+
|
|
28
|
+
Parameters
|
|
29
|
+
----------
|
|
30
|
+
column : str
|
|
31
|
+
Name of the column to be visualized.
|
|
32
|
+
|
|
33
|
+
top_n_categories : int, default=15
|
|
34
|
+
Number of most frequent categories to display for categorical data.
|
|
35
|
+
|
|
36
|
+
histogram_bins : int, default=15
|
|
37
|
+
Number of bins used in histogram visualizations for numeric data.
|
|
38
|
+
|
|
39
|
+
Notes
|
|
40
|
+
-----
|
|
41
|
+
Plotly operates locally, so aggregated results are converted to Pandas
|
|
42
|
+
before constructing the figures. The implementation minimizes the amount
|
|
43
|
+
of data transferred to the driver by performing filtering, aggregation
|
|
44
|
+
and binning with Spark whenever possible.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
column: str,
|
|
50
|
+
top_n_categories: int = 15,
|
|
51
|
+
histogram_bins: int = 15
|
|
52
|
+
):
|
|
53
|
+
"""
|
|
54
|
+
Initializes the visualization object.
|
|
55
|
+
|
|
56
|
+
Parameters
|
|
57
|
+
----------
|
|
58
|
+
column : str
|
|
59
|
+
Name of the column to be visualized.
|
|
60
|
+
|
|
61
|
+
top_n_categories : int, default=15
|
|
62
|
+
Number of most frequent categories to display for categorical
|
|
63
|
+
data.
|
|
64
|
+
|
|
65
|
+
histogram_bins : int, default=15
|
|
66
|
+
Number of bins used in histogram visualizations for numeric data.
|
|
67
|
+
|
|
68
|
+
Raises
|
|
69
|
+
------
|
|
70
|
+
ValueError
|
|
71
|
+
If ``column`` is not a non-empty string, if
|
|
72
|
+
``top_n_categories`` is not an integer greater than 1, or if
|
|
73
|
+
``histogram_bins`` is not a positive integer.
|
|
74
|
+
"""
|
|
75
|
+
if not isinstance(column, str) or not column.strip():
|
|
76
|
+
raise ValueError(
|
|
77
|
+
"'column' must be a non-empty string."
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
if (
|
|
81
|
+
not isinstance(top_n_categories, int)
|
|
82
|
+
or isinstance(top_n_categories, bool)
|
|
83
|
+
):
|
|
84
|
+
raise ValueError(
|
|
85
|
+
"'top_n_categories' must be an integer."
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
if top_n_categories <= 1:
|
|
89
|
+
raise ValueError(
|
|
90
|
+
"'top_n_categories' must be greater than 1."
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
if (
|
|
94
|
+
not isinstance(histogram_bins, int)
|
|
95
|
+
or isinstance(histogram_bins, bool)
|
|
96
|
+
):
|
|
97
|
+
raise ValueError(
|
|
98
|
+
"'histogram_bins' must be an integer."
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
if histogram_bins <= 0:
|
|
102
|
+
raise ValueError(
|
|
103
|
+
"'histogram_bins' must be greater than 0."
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
self.column = column
|
|
107
|
+
self.top_n_categories = top_n_categories
|
|
108
|
+
self.histogram_bins = histogram_bins
|
|
109
|
+
|
|
110
|
+
def _add_toggle_menu(
|
|
111
|
+
self,
|
|
112
|
+
fig: go.Figure,
|
|
113
|
+
labels: list[str]
|
|
114
|
+
) -> None:
|
|
115
|
+
"""
|
|
116
|
+
Adds an interactive menu to switch between Plotly traces.
|
|
117
|
+
|
|
118
|
+
Each button makes one trace visible and hides the remaining traces,
|
|
119
|
+
allowing several visualization types to be stored in the same
|
|
120
|
+
Plotly figure.
|
|
121
|
+
|
|
122
|
+
Parameters
|
|
123
|
+
----------
|
|
124
|
+
fig : plotly.graph_objects.Figure
|
|
125
|
+
Figure containing the traces that will be controlled by the
|
|
126
|
+
toggle menu.
|
|
127
|
+
|
|
128
|
+
labels : list of str
|
|
129
|
+
Labels displayed in the interactive menu. The order of the
|
|
130
|
+
labels must correspond to the order of the traces in the figure.
|
|
131
|
+
|
|
132
|
+
Returns
|
|
133
|
+
-------
|
|
134
|
+
None
|
|
135
|
+
The figure is modified in place.
|
|
136
|
+
"""
|
|
137
|
+
buttons = []
|
|
138
|
+
n = len(fig.data)
|
|
139
|
+
|
|
140
|
+
for i, label in enumerate(labels):
|
|
141
|
+
visible = [False] * n
|
|
142
|
+
visible[i] = True
|
|
143
|
+
|
|
144
|
+
buttons.append(
|
|
145
|
+
{
|
|
146
|
+
"label": label,
|
|
147
|
+
"method": "update",
|
|
148
|
+
"args": [
|
|
149
|
+
{"visible": visible},
|
|
150
|
+
{"title": fig.layout.title.text}
|
|
151
|
+
],
|
|
152
|
+
}
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
fig.update_layout(
|
|
156
|
+
updatemenus=[
|
|
157
|
+
{
|
|
158
|
+
"type": "buttons",
|
|
159
|
+
"direction": "right",
|
|
160
|
+
"buttons": buttons,
|
|
161
|
+
"x": 0.5,
|
|
162
|
+
"xanchor": "center",
|
|
163
|
+
"y": 1.18,
|
|
164
|
+
"yanchor": "top",
|
|
165
|
+
"showactive": True,
|
|
166
|
+
}
|
|
167
|
+
]
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
def _build_numeric_figure(
|
|
171
|
+
self,
|
|
172
|
+
df: DataFrame
|
|
173
|
+
) -> go.Figure:
|
|
174
|
+
"""
|
|
175
|
+
Builds interactive visualizations for a numeric column.
|
|
176
|
+
|
|
177
|
+
Three visualization types are generated:
|
|
178
|
+
|
|
179
|
+
- Barplot:
|
|
180
|
+
Displays the frequency distribution of the numeric values when
|
|
181
|
+
the number of distinct values is small enough to visualize
|
|
182
|
+
individually.
|
|
183
|
+
|
|
184
|
+
When the number of distinct values exceeds
|
|
185
|
+
``top_n_categories``, the most frequent values are displayed
|
|
186
|
+
instead of transferring the complete frequency distribution
|
|
187
|
+
to the driver.
|
|
188
|
+
|
|
189
|
+
- Histogram:
|
|
190
|
+
Displays the distribution using ``histogram_bins`` bins.
|
|
191
|
+
Histogram binning is performed by Spark so that only aggregated
|
|
192
|
+
bin counts are transferred to the driver.
|
|
193
|
+
|
|
194
|
+
- Boxplot:
|
|
195
|
+
Displays approximate quartiles together with the minimum,
|
|
196
|
+
maximum and mean values.
|
|
197
|
+
|
|
198
|
+
Parameters
|
|
199
|
+
----------
|
|
200
|
+
df : pyspark.sql.DataFrame
|
|
201
|
+
Input Spark DataFrame containing the numeric column.
|
|
202
|
+
|
|
203
|
+
Returns
|
|
204
|
+
-------
|
|
205
|
+
plotly.graph_objects.Figure
|
|
206
|
+
Interactive Plotly figure containing the numeric
|
|
207
|
+
visualizations.
|
|
208
|
+
|
|
209
|
+
Raises
|
|
210
|
+
------
|
|
211
|
+
ValueError
|
|
212
|
+
If the selected column contains no valid numeric values.
|
|
213
|
+
|
|
214
|
+
Notes
|
|
215
|
+
-----
|
|
216
|
+
The method avoids collecting all distinct numeric values to the
|
|
217
|
+
driver. Aggregation and histogram binning are performed with Spark,
|
|
218
|
+
making the visualization suitable for high-cardinality numeric
|
|
219
|
+
columns and large datasets.
|
|
220
|
+
|
|
221
|
+
Quartiles are calculated using Spark's approximate quantile
|
|
222
|
+
algorithm with a relative error of 0.01.
|
|
223
|
+
"""
|
|
224
|
+
values_df = (
|
|
225
|
+
df
|
|
226
|
+
.select(
|
|
227
|
+
F.col(self.column)
|
|
228
|
+
.cast("double")
|
|
229
|
+
.alias(self.column)
|
|
230
|
+
)
|
|
231
|
+
.dropna()
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
# ------------------------------------------------------
|
|
235
|
+
# BASIC STATISTICS
|
|
236
|
+
# ------------------------------------------------------
|
|
237
|
+
|
|
238
|
+
stats = (
|
|
239
|
+
values_df
|
|
240
|
+
.agg(
|
|
241
|
+
F.min(self.column).alias("min_value"),
|
|
242
|
+
F.max(self.column).alias("max_value"),
|
|
243
|
+
F.mean(self.column).alias("mean_value"),
|
|
244
|
+
F.count("*").alias("count")
|
|
245
|
+
)
|
|
246
|
+
.first()
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
if stats["count"] == 0:
|
|
250
|
+
raise ValueError(
|
|
251
|
+
f"Column '{self.column}' contains no valid numeric values."
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
min_value = stats["min_value"]
|
|
255
|
+
max_value = stats["max_value"]
|
|
256
|
+
mean_value = stats["mean_value"]
|
|
257
|
+
|
|
258
|
+
# ------------------------------------------------------
|
|
259
|
+
# QUARTILES
|
|
260
|
+
# ------------------------------------------------------
|
|
261
|
+
|
|
262
|
+
q1, median, q3 = values_df.approxQuantile(
|
|
263
|
+
self.column,
|
|
264
|
+
[0.25, 0.5, 0.75],
|
|
265
|
+
0.01
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
# ------------------------------------------------------
|
|
269
|
+
# BARPLOT
|
|
270
|
+
# ------------------------------------------------------
|
|
271
|
+
#
|
|
272
|
+
# Only a bounded number of values is collected to the
|
|
273
|
+
# driver. This prevents high-cardinality numeric columns
|
|
274
|
+
# from generating very large Pandas DataFrames.
|
|
275
|
+
# ------------------------------------------------------
|
|
276
|
+
|
|
277
|
+
bar_counts = (
|
|
278
|
+
values_df
|
|
279
|
+
.groupBy(self.column)
|
|
280
|
+
.agg(
|
|
281
|
+
F.count("*").alias("frequency")
|
|
282
|
+
)
|
|
283
|
+
.orderBy(
|
|
284
|
+
F.desc("frequency"),
|
|
285
|
+
F.asc(self.column)
|
|
286
|
+
)
|
|
287
|
+
.limit(self.top_n_categories)
|
|
288
|
+
.toPandas()
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
# Sort selected values numerically for a more natural
|
|
292
|
+
# representation in the barplot.
|
|
293
|
+
bar_counts = bar_counts.sort_values(
|
|
294
|
+
self.column
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
# ------------------------------------------------------
|
|
298
|
+
# HISTOGRAM
|
|
299
|
+
# ------------------------------------------------------
|
|
300
|
+
|
|
301
|
+
if min_value == max_value:
|
|
302
|
+
histogram_df = (
|
|
303
|
+
values_df
|
|
304
|
+
.agg(
|
|
305
|
+
F.count("*").alias("frequency")
|
|
306
|
+
)
|
|
307
|
+
.withColumn(
|
|
308
|
+
"bin_center",
|
|
309
|
+
F.lit(float(min_value))
|
|
310
|
+
)
|
|
311
|
+
.select(
|
|
312
|
+
"bin_center",
|
|
313
|
+
"frequency"
|
|
314
|
+
)
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
bin_width = 1.0
|
|
318
|
+
|
|
319
|
+
else:
|
|
320
|
+
bin_width = (
|
|
321
|
+
(max_value - min_value)
|
|
322
|
+
/ self.histogram_bins
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
bin_index = F.floor(
|
|
326
|
+
(F.col(self.column) - F.lit(min_value))
|
|
327
|
+
/ F.lit(bin_width)
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
# The maximum value would otherwise fall into bin
|
|
331
|
+
# histogram_bins instead of histogram_bins - 1.
|
|
332
|
+
bin_index = F.when(
|
|
333
|
+
bin_index >= self.histogram_bins,
|
|
334
|
+
F.lit(self.histogram_bins - 1)
|
|
335
|
+
).otherwise(bin_index)
|
|
336
|
+
|
|
337
|
+
histogram_df = (
|
|
338
|
+
values_df
|
|
339
|
+
.withColumn(
|
|
340
|
+
"_bin",
|
|
341
|
+
bin_index.cast("int")
|
|
342
|
+
)
|
|
343
|
+
.groupBy("_bin")
|
|
344
|
+
.agg(
|
|
345
|
+
F.count("*").alias("frequency")
|
|
346
|
+
)
|
|
347
|
+
.withColumn(
|
|
348
|
+
"bin_center",
|
|
349
|
+
F.lit(min_value)
|
|
350
|
+
+ (
|
|
351
|
+
F.col("_bin") + F.lit(0.5)
|
|
352
|
+
) * F.lit(bin_width)
|
|
353
|
+
)
|
|
354
|
+
.select(
|
|
355
|
+
"bin_center",
|
|
356
|
+
"frequency"
|
|
357
|
+
)
|
|
358
|
+
.orderBy("bin_center")
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
histogram = histogram_df.toPandas()
|
|
362
|
+
|
|
363
|
+
# ------------------------------------------------------
|
|
364
|
+
# FIGURE
|
|
365
|
+
# ------------------------------------------------------
|
|
366
|
+
|
|
367
|
+
fig = go.Figure()
|
|
368
|
+
|
|
369
|
+
fig.add_trace(
|
|
370
|
+
go.Bar(
|
|
371
|
+
x=bar_counts[self.column],
|
|
372
|
+
y=bar_counts["frequency"],
|
|
373
|
+
name="Barplot"
|
|
374
|
+
)
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
fig.add_trace(
|
|
378
|
+
go.Bar(
|
|
379
|
+
x=histogram["bin_center"],
|
|
380
|
+
y=histogram["frequency"],
|
|
381
|
+
width=bin_width,
|
|
382
|
+
name="Histogram",
|
|
383
|
+
visible=False
|
|
384
|
+
)
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
fig.add_trace(
|
|
388
|
+
go.Box(
|
|
389
|
+
q1=[q1],
|
|
390
|
+
median=[median],
|
|
391
|
+
mean=[mean_value],
|
|
392
|
+
q3=[q3],
|
|
393
|
+
lowerfence=[min_value],
|
|
394
|
+
upperfence=[max_value],
|
|
395
|
+
name="Boxplot",
|
|
396
|
+
visible=False,
|
|
397
|
+
boxpoints=False
|
|
398
|
+
)
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
fig.update_layout(
|
|
402
|
+
title=(
|
|
403
|
+
f"{self.column.title()} "
|
|
404
|
+
f"\t|\t(histogram_bins={self.histogram_bins})"
|
|
405
|
+
),
|
|
406
|
+
xaxis_title=self.column,
|
|
407
|
+
yaxis_title="Count",
|
|
408
|
+
legend_title="Chart",
|
|
409
|
+
template="plotly_white"
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
self._add_toggle_menu(
|
|
413
|
+
fig,
|
|
414
|
+
["Barplot", "Histogram", "Boxplot"]
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
return fig
|
|
418
|
+
|
|
419
|
+
def _build_categorical_figure(
|
|
420
|
+
self,
|
|
421
|
+
df: DataFrame
|
|
422
|
+
) -> go.Figure:
|
|
423
|
+
"""
|
|
424
|
+
Builds interactive visualizations for a categorical column.
|
|
425
|
+
|
|
426
|
+
Two visualization types are generated:
|
|
427
|
+
|
|
428
|
+
- Bar:
|
|
429
|
+
Displays the frequency of the most common categories.
|
|
430
|
+
|
|
431
|
+
- Treemap:
|
|
432
|
+
Displays the same category frequencies using a hierarchical
|
|
433
|
+
area-based representation.
|
|
434
|
+
|
|
435
|
+
Only the ``top_n_categories`` most frequent values are transferred
|
|
436
|
+
from Spark to the driver.
|
|
437
|
+
|
|
438
|
+
Parameters
|
|
439
|
+
----------
|
|
440
|
+
df : pyspark.sql.DataFrame
|
|
441
|
+
Input Spark DataFrame containing the categorical column.
|
|
442
|
+
|
|
443
|
+
Returns
|
|
444
|
+
-------
|
|
445
|
+
plotly.graph_objects.Figure
|
|
446
|
+
Interactive Plotly figure containing the categorical
|
|
447
|
+
visualizations.
|
|
448
|
+
|
|
449
|
+
Raises
|
|
450
|
+
------
|
|
451
|
+
ValueError
|
|
452
|
+
If the selected column contains no non-null values.
|
|
453
|
+
|
|
454
|
+
Notes
|
|
455
|
+
-----
|
|
456
|
+
Aggregation, ordering and limiting are performed by Spark before
|
|
457
|
+
conversion to Pandas. Therefore, the amount of data transferred
|
|
458
|
+
to the driver is bounded by ``top_n_categories``.
|
|
459
|
+
"""
|
|
460
|
+
counts_df = (
|
|
461
|
+
df
|
|
462
|
+
.select(self.column)
|
|
463
|
+
.dropna()
|
|
464
|
+
.withColumn(
|
|
465
|
+
self.column,
|
|
466
|
+
F.col(self.column).cast("string")
|
|
467
|
+
)
|
|
468
|
+
.groupBy(self.column)
|
|
469
|
+
.agg(
|
|
470
|
+
F.count("*").alias("count")
|
|
471
|
+
)
|
|
472
|
+
.orderBy(
|
|
473
|
+
F.desc("count")
|
|
474
|
+
)
|
|
475
|
+
.limit(
|
|
476
|
+
self.top_n_categories
|
|
477
|
+
)
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
counts = counts_df.toPandas()
|
|
481
|
+
|
|
482
|
+
if counts.empty:
|
|
483
|
+
raise ValueError(
|
|
484
|
+
f"Column '{self.column}' contains no non-null values."
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
fig = go.Figure()
|
|
488
|
+
|
|
489
|
+
fig.add_trace(
|
|
490
|
+
go.Bar(
|
|
491
|
+
x=counts[self.column],
|
|
492
|
+
y=counts["count"],
|
|
493
|
+
name="Bar",
|
|
494
|
+
visible=True
|
|
495
|
+
)
|
|
496
|
+
)
|
|
497
|
+
|
|
498
|
+
fig.add_trace(
|
|
499
|
+
go.Treemap(
|
|
500
|
+
labels=counts[self.column],
|
|
501
|
+
parents=[""] * len(counts),
|
|
502
|
+
values=counts["count"],
|
|
503
|
+
name="Treemap",
|
|
504
|
+
visible=False
|
|
505
|
+
)
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
fig.update_layout(
|
|
509
|
+
title=(
|
|
510
|
+
f"{self.column.title()}"
|
|
511
|
+
f"\t|\t(top_n_categories="
|
|
512
|
+
f"{self.top_n_categories})"
|
|
513
|
+
),
|
|
514
|
+
xaxis_title=self.column,
|
|
515
|
+
yaxis_title="Count",
|
|
516
|
+
legend_title="Chart",
|
|
517
|
+
template="plotly_white"
|
|
518
|
+
)
|
|
519
|
+
|
|
520
|
+
self._add_toggle_menu(
|
|
521
|
+
fig,
|
|
522
|
+
["Bar", "Treemap"]
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
return fig
|
|
526
|
+
|
|
527
|
+
def _build_date_figure(
|
|
528
|
+
self,
|
|
529
|
+
df: DataFrame
|
|
530
|
+
) -> go.Figure:
|
|
531
|
+
"""
|
|
532
|
+
Builds interactive visualizations for a date column.
|
|
533
|
+
|
|
534
|
+
Three visualization types are generated:
|
|
535
|
+
|
|
536
|
+
- Year:
|
|
537
|
+
Displays the number of records for each year.
|
|
538
|
+
|
|
539
|
+
- Month:
|
|
540
|
+
Displays the number of records for each calendar month,
|
|
541
|
+
aggregated across all years.
|
|
542
|
+
|
|
543
|
+
- Sunburst:
|
|
544
|
+
Displays the hierarchical year-month distribution.
|
|
545
|
+
|
|
546
|
+
Parameters
|
|
547
|
+
----------
|
|
548
|
+
df : pyspark.sql.DataFrame
|
|
549
|
+
Input Spark DataFrame containing the date column.
|
|
550
|
+
|
|
551
|
+
Returns
|
|
552
|
+
-------
|
|
553
|
+
plotly.graph_objects.Figure
|
|
554
|
+
Interactive Plotly figure containing the date
|
|
555
|
+
visualizations.
|
|
556
|
+
|
|
557
|
+
Raises
|
|
558
|
+
------
|
|
559
|
+
ValueError
|
|
560
|
+
If the selected column contains no valid date values.
|
|
561
|
+
|
|
562
|
+
Notes
|
|
563
|
+
-----
|
|
564
|
+
Spark performs a single aggregation by year and month. The
|
|
565
|
+
resulting DataFrame contains at most one row for each observed
|
|
566
|
+
year-month combination and is therefore small enough to convert
|
|
567
|
+
to Pandas.
|
|
568
|
+
|
|
569
|
+
Year-level and month-level distributions are subsequently derived
|
|
570
|
+
locally from this already aggregated result, avoiding multiple
|
|
571
|
+
Spark groupBy operations over the original dataset.
|
|
572
|
+
"""
|
|
573
|
+
date_df = (
|
|
574
|
+
df
|
|
575
|
+
.select(
|
|
576
|
+
F.to_date(
|
|
577
|
+
F.col(self.column)
|
|
578
|
+
).alias(self.column)
|
|
579
|
+
)
|
|
580
|
+
.dropna()
|
|
581
|
+
.select(
|
|
582
|
+
F.year(
|
|
583
|
+
F.col(self.column)
|
|
584
|
+
).alias("year"),
|
|
585
|
+
F.month(
|
|
586
|
+
F.col(self.column)
|
|
587
|
+
).alias("month")
|
|
588
|
+
)
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
# ------------------------------------------------------
|
|
592
|
+
# SINGLE SPARK AGGREGATION
|
|
593
|
+
# ------------------------------------------------------
|
|
594
|
+
|
|
595
|
+
year_month_counts = (
|
|
596
|
+
date_df
|
|
597
|
+
.groupBy(
|
|
598
|
+
"year",
|
|
599
|
+
"month"
|
|
600
|
+
)
|
|
601
|
+
.agg(
|
|
602
|
+
F.count("*").alias("count")
|
|
603
|
+
)
|
|
604
|
+
.orderBy(
|
|
605
|
+
"year",
|
|
606
|
+
"month"
|
|
607
|
+
)
|
|
608
|
+
.toPandas()
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
if year_month_counts.empty:
|
|
612
|
+
raise ValueError(
|
|
613
|
+
f"Column '{self.column}' contains no valid date values."
|
|
614
|
+
)
|
|
615
|
+
|
|
616
|
+
# ------------------------------------------------------
|
|
617
|
+
# LOCAL AGGREGATIONS
|
|
618
|
+
# ------------------------------------------------------
|
|
619
|
+
|
|
620
|
+
year_counts = (
|
|
621
|
+
year_month_counts
|
|
622
|
+
.groupby(
|
|
623
|
+
"year",
|
|
624
|
+
as_index=False
|
|
625
|
+
)["count"]
|
|
626
|
+
.sum()
|
|
627
|
+
.sort_values("year")
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
month_counts = (
|
|
631
|
+
year_month_counts
|
|
632
|
+
.groupby(
|
|
633
|
+
"month",
|
|
634
|
+
as_index=False
|
|
635
|
+
)["count"]
|
|
636
|
+
.sum()
|
|
637
|
+
.sort_values("month")
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
month_nodes = year_month_counts.copy()
|
|
641
|
+
|
|
642
|
+
month_names = {
|
|
643
|
+
1: "January",
|
|
644
|
+
2: "February",
|
|
645
|
+
3: "March",
|
|
646
|
+
4: "April",
|
|
647
|
+
5: "May",
|
|
648
|
+
6: "June",
|
|
649
|
+
7: "July",
|
|
650
|
+
8: "August",
|
|
651
|
+
9: "September",
|
|
652
|
+
10: "October",
|
|
653
|
+
11: "November",
|
|
654
|
+
12: "December"
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
month_counts["month_name"] = (
|
|
658
|
+
month_counts["month"]
|
|
659
|
+
.map(month_names)
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
# ------------------------------------------------------
|
|
663
|
+
# SUNBURST NODES
|
|
664
|
+
# ------------------------------------------------------
|
|
665
|
+
|
|
666
|
+
year_nodes = year_counts.copy()
|
|
667
|
+
|
|
668
|
+
year_nodes["id"] = (
|
|
669
|
+
year_nodes["year"]
|
|
670
|
+
.astype(str)
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
year_nodes["label"] = (
|
|
674
|
+
year_nodes["year"]
|
|
675
|
+
.astype(str)
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
year_nodes["parent"] = ""
|
|
679
|
+
|
|
680
|
+
month_nodes["id"] = (
|
|
681
|
+
month_nodes["year"].astype(str)
|
|
682
|
+
+ "-"
|
|
683
|
+
+ month_nodes["month"]
|
|
684
|
+
.astype(str)
|
|
685
|
+
.str.zfill(2)
|
|
686
|
+
)
|
|
687
|
+
|
|
688
|
+
month_nodes["label"] = (
|
|
689
|
+
month_nodes["month"]
|
|
690
|
+
.map(month_names)
|
|
691
|
+
)
|
|
692
|
+
|
|
693
|
+
month_nodes["parent"] = (
|
|
694
|
+
month_nodes["year"]
|
|
695
|
+
.astype(str)
|
|
696
|
+
)
|
|
697
|
+
|
|
698
|
+
sunburst_ids = (
|
|
699
|
+
year_nodes["id"].tolist()
|
|
700
|
+
+ month_nodes["id"].tolist()
|
|
701
|
+
)
|
|
702
|
+
|
|
703
|
+
sunburst_labels = (
|
|
704
|
+
year_nodes["label"].tolist()
|
|
705
|
+
+ month_nodes["label"].tolist()
|
|
706
|
+
)
|
|
707
|
+
|
|
708
|
+
sunburst_parents = (
|
|
709
|
+
year_nodes["parent"].tolist()
|
|
710
|
+
+ month_nodes["parent"].tolist()
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
sunburst_values = (
|
|
714
|
+
year_nodes["count"].tolist()
|
|
715
|
+
+ month_nodes["count"].tolist()
|
|
716
|
+
)
|
|
717
|
+
|
|
718
|
+
# ------------------------------------------------------
|
|
719
|
+
# FIGURE
|
|
720
|
+
# ------------------------------------------------------
|
|
721
|
+
|
|
722
|
+
fig = go.Figure()
|
|
723
|
+
|
|
724
|
+
fig.add_trace(
|
|
725
|
+
go.Bar(
|
|
726
|
+
x=year_counts["year"].astype(str),
|
|
727
|
+
y=year_counts["count"],
|
|
728
|
+
name="Year",
|
|
729
|
+
visible=True
|
|
730
|
+
)
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
fig.add_trace(
|
|
734
|
+
go.Bar(
|
|
735
|
+
x=month_counts["month_name"],
|
|
736
|
+
y=month_counts["count"],
|
|
737
|
+
name="Month",
|
|
738
|
+
visible=False
|
|
739
|
+
)
|
|
740
|
+
)
|
|
741
|
+
|
|
742
|
+
fig.add_trace(
|
|
743
|
+
go.Sunburst(
|
|
744
|
+
ids=sunburst_ids,
|
|
745
|
+
labels=sunburst_labels,
|
|
746
|
+
parents=sunburst_parents,
|
|
747
|
+
values=sunburst_values,
|
|
748
|
+
branchvalues="total",
|
|
749
|
+
name="Sunburst",
|
|
750
|
+
visible=False,
|
|
751
|
+
sort=False
|
|
752
|
+
)
|
|
753
|
+
)
|
|
754
|
+
|
|
755
|
+
fig.update_layout(
|
|
756
|
+
title=f"{self.column.title()}",
|
|
757
|
+
xaxis_title="Date",
|
|
758
|
+
yaxis_title="Count",
|
|
759
|
+
legend_title="Chart",
|
|
760
|
+
template="plotly_white"
|
|
761
|
+
)
|
|
762
|
+
|
|
763
|
+
self._add_toggle_menu(
|
|
764
|
+
fig,
|
|
765
|
+
["Year", "Month", "Sunburst"]
|
|
766
|
+
)
|
|
767
|
+
|
|
768
|
+
return fig
|
|
769
|
+
|
|
770
|
+
def transform(
|
|
771
|
+
self,
|
|
772
|
+
df: DataFrame
|
|
773
|
+
) -> go.Figure:
|
|
774
|
+
"""
|
|
775
|
+
Generates an interactive visualization for the configured column.
|
|
776
|
+
|
|
777
|
+
The semantic type of the selected column is inferred automatically
|
|
778
|
+
and the appropriate visualization strategy is applied.
|
|
779
|
+
|
|
780
|
+
Numeric columns are visualized using a barplot, histogram and
|
|
781
|
+
boxplot. Date columns are visualized using yearly, monthly and
|
|
782
|
+
hierarchical year-month distributions. Other columns are treated
|
|
783
|
+
as categorical and visualized using a bar chart and treemap.
|
|
784
|
+
|
|
785
|
+
Parameters
|
|
786
|
+
----------
|
|
787
|
+
df : pyspark.sql.DataFrame
|
|
788
|
+
Input Spark DataFrame containing the column to visualize.
|
|
789
|
+
|
|
790
|
+
Returns
|
|
791
|
+
-------
|
|
792
|
+
plotly.graph_objects.Figure
|
|
793
|
+
Interactive Plotly figure corresponding to the inferred
|
|
794
|
+
semantic type of the selected column.
|
|
795
|
+
|
|
796
|
+
Raises
|
|
797
|
+
------
|
|
798
|
+
ValueError
|
|
799
|
+
If the input DataFrame is None, is not Spark-compatible, or
|
|
800
|
+
does not contain the configured column.
|
|
801
|
+
|
|
802
|
+
Notes
|
|
803
|
+
-----
|
|
804
|
+
The method delegates the visualization process to one of the
|
|
805
|
+
specialized internal methods:
|
|
806
|
+
|
|
807
|
+
- ``_build_numeric_figure``
|
|
808
|
+
- ``_build_date_figure``
|
|
809
|
+
- ``_build_categorical_figure``
|
|
810
|
+
|
|
811
|
+
Data-intensive operations are performed with Spark before the
|
|
812
|
+
aggregated results are transferred to the driver for Plotly
|
|
813
|
+
visualization.
|
|
814
|
+
"""
|
|
815
|
+
if df is None:
|
|
816
|
+
raise ValueError(
|
|
817
|
+
"Input DataFrame cannot be None."
|
|
818
|
+
)
|
|
819
|
+
|
|
820
|
+
# Compatible with standard Spark and Spark Connect.
|
|
821
|
+
if not hasattr(df, "select"):
|
|
822
|
+
raise ValueError(
|
|
823
|
+
"Input must be a Spark-compatible DataFrame."
|
|
824
|
+
)
|
|
825
|
+
|
|
826
|
+
if self.column not in df.columns:
|
|
827
|
+
raise ValueError(
|
|
828
|
+
f"Column '{self.column}' not found in DataFrame."
|
|
829
|
+
)
|
|
830
|
+
|
|
831
|
+
semantic_type = infer_semantic_type(
|
|
832
|
+
df,
|
|
833
|
+
self.column
|
|
834
|
+
)
|
|
835
|
+
|
|
836
|
+
if semantic_type == "numeric":
|
|
837
|
+
fig = self._build_numeric_figure(df)
|
|
838
|
+
|
|
839
|
+
elif semantic_type == "date":
|
|
840
|
+
fig = self._build_date_figure(df)
|
|
841
|
+
|
|
842
|
+
else:
|
|
843
|
+
fig = self._build_categorical_figure(df)
|
|
844
|
+
|
|
845
|
+
return fig
|