anonymization-lib 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,845 @@
1
+ from pyspark.sql import DataFrame
2
+ from pyspark.sql import functions as F
3
+ import plotly.graph_objects as go
4
+
5
+ from ..utils import infer_semantic_type
6
+
7
+
8
+ class Visualization:
9
+ """
10
+ Generates interactive visualizations for a given column in a Spark DataFrame.
11
+
12
+ The visualization generated depends on the semantic type inferred for the
13
+ selected column:
14
+
15
+ - Numeric columns:
16
+ Bar plot, histogram and box plot.
17
+
18
+ - Categorical columns:
19
+ Bar plot and treemap containing the most frequent categories.
20
+
21
+ - Date columns:
22
+ Year distribution, month distribution and year-month sunburst.
23
+
24
+ The class is designed to perform data-intensive operations using Spark
25
+ before transferring aggregated results to the driver for visualization
26
+ with Plotly.
27
+
28
+ Parameters
29
+ ----------
30
+ column : str
31
+ Name of the column to be visualized.
32
+
33
+ top_n_categories : int, default=15
34
+ Number of most frequent categories to display for categorical data.
35
+
36
+ histogram_bins : int, default=15
37
+ Number of bins used in histogram visualizations for numeric data.
38
+
39
+ Notes
40
+ -----
41
+ Plotly operates locally, so aggregated results are converted to Pandas
42
+ before constructing the figures. The implementation minimizes the amount
43
+ of data transferred to the driver by performing filtering, aggregation
44
+ and binning with Spark whenever possible.
45
+ """
46
+
47
+ def __init__(
48
+ self,
49
+ column: str,
50
+ top_n_categories: int = 15,
51
+ histogram_bins: int = 15
52
+ ):
53
+ """
54
+ Initializes the visualization object.
55
+
56
+ Parameters
57
+ ----------
58
+ column : str
59
+ Name of the column to be visualized.
60
+
61
+ top_n_categories : int, default=15
62
+ Number of most frequent categories to display for categorical
63
+ data.
64
+
65
+ histogram_bins : int, default=15
66
+ Number of bins used in histogram visualizations for numeric data.
67
+
68
+ Raises
69
+ ------
70
+ ValueError
71
+ If ``column`` is not a non-empty string, if
72
+ ``top_n_categories`` is not an integer greater than 1, or if
73
+ ``histogram_bins`` is not a positive integer.
74
+ """
75
+ if not isinstance(column, str) or not column.strip():
76
+ raise ValueError(
77
+ "'column' must be a non-empty string."
78
+ )
79
+
80
+ if (
81
+ not isinstance(top_n_categories, int)
82
+ or isinstance(top_n_categories, bool)
83
+ ):
84
+ raise ValueError(
85
+ "'top_n_categories' must be an integer."
86
+ )
87
+
88
+ if top_n_categories <= 1:
89
+ raise ValueError(
90
+ "'top_n_categories' must be greater than 1."
91
+ )
92
+
93
+ if (
94
+ not isinstance(histogram_bins, int)
95
+ or isinstance(histogram_bins, bool)
96
+ ):
97
+ raise ValueError(
98
+ "'histogram_bins' must be an integer."
99
+ )
100
+
101
+ if histogram_bins <= 0:
102
+ raise ValueError(
103
+ "'histogram_bins' must be greater than 0."
104
+ )
105
+
106
+ self.column = column
107
+ self.top_n_categories = top_n_categories
108
+ self.histogram_bins = histogram_bins
109
+
110
+ def _add_toggle_menu(
111
+ self,
112
+ fig: go.Figure,
113
+ labels: list[str]
114
+ ) -> None:
115
+ """
116
+ Adds an interactive menu to switch between Plotly traces.
117
+
118
+ Each button makes one trace visible and hides the remaining traces,
119
+ allowing several visualization types to be stored in the same
120
+ Plotly figure.
121
+
122
+ Parameters
123
+ ----------
124
+ fig : plotly.graph_objects.Figure
125
+ Figure containing the traces that will be controlled by the
126
+ toggle menu.
127
+
128
+ labels : list of str
129
+ Labels displayed in the interactive menu. The order of the
130
+ labels must correspond to the order of the traces in the figure.
131
+
132
+ Returns
133
+ -------
134
+ None
135
+ The figure is modified in place.
136
+ """
137
+ buttons = []
138
+ n = len(fig.data)
139
+
140
+ for i, label in enumerate(labels):
141
+ visible = [False] * n
142
+ visible[i] = True
143
+
144
+ buttons.append(
145
+ {
146
+ "label": label,
147
+ "method": "update",
148
+ "args": [
149
+ {"visible": visible},
150
+ {"title": fig.layout.title.text}
151
+ ],
152
+ }
153
+ )
154
+
155
+ fig.update_layout(
156
+ updatemenus=[
157
+ {
158
+ "type": "buttons",
159
+ "direction": "right",
160
+ "buttons": buttons,
161
+ "x": 0.5,
162
+ "xanchor": "center",
163
+ "y": 1.18,
164
+ "yanchor": "top",
165
+ "showactive": True,
166
+ }
167
+ ]
168
+ )
169
+
170
+ def _build_numeric_figure(
171
+ self,
172
+ df: DataFrame
173
+ ) -> go.Figure:
174
+ """
175
+ Builds interactive visualizations for a numeric column.
176
+
177
+ Three visualization types are generated:
178
+
179
+ - Barplot:
180
+ Displays the frequency distribution of the numeric values when
181
+ the number of distinct values is small enough to visualize
182
+ individually.
183
+
184
+ When the number of distinct values exceeds
185
+ ``top_n_categories``, the most frequent values are displayed
186
+ instead of transferring the complete frequency distribution
187
+ to the driver.
188
+
189
+ - Histogram:
190
+ Displays the distribution using ``histogram_bins`` bins.
191
+ Histogram binning is performed by Spark so that only aggregated
192
+ bin counts are transferred to the driver.
193
+
194
+ - Boxplot:
195
+ Displays approximate quartiles together with the minimum,
196
+ maximum and mean values.
197
+
198
+ Parameters
199
+ ----------
200
+ df : pyspark.sql.DataFrame
201
+ Input Spark DataFrame containing the numeric column.
202
+
203
+ Returns
204
+ -------
205
+ plotly.graph_objects.Figure
206
+ Interactive Plotly figure containing the numeric
207
+ visualizations.
208
+
209
+ Raises
210
+ ------
211
+ ValueError
212
+ If the selected column contains no valid numeric values.
213
+
214
+ Notes
215
+ -----
216
+ The method avoids collecting all distinct numeric values to the
217
+ driver. Aggregation and histogram binning are performed with Spark,
218
+ making the visualization suitable for high-cardinality numeric
219
+ columns and large datasets.
220
+
221
+ Quartiles are calculated using Spark's approximate quantile
222
+ algorithm with a relative error of 0.01.
223
+ """
224
+ values_df = (
225
+ df
226
+ .select(
227
+ F.col(self.column)
228
+ .cast("double")
229
+ .alias(self.column)
230
+ )
231
+ .dropna()
232
+ )
233
+
234
+ # ------------------------------------------------------
235
+ # BASIC STATISTICS
236
+ # ------------------------------------------------------
237
+
238
+ stats = (
239
+ values_df
240
+ .agg(
241
+ F.min(self.column).alias("min_value"),
242
+ F.max(self.column).alias("max_value"),
243
+ F.mean(self.column).alias("mean_value"),
244
+ F.count("*").alias("count")
245
+ )
246
+ .first()
247
+ )
248
+
249
+ if stats["count"] == 0:
250
+ raise ValueError(
251
+ f"Column '{self.column}' contains no valid numeric values."
252
+ )
253
+
254
+ min_value = stats["min_value"]
255
+ max_value = stats["max_value"]
256
+ mean_value = stats["mean_value"]
257
+
258
+ # ------------------------------------------------------
259
+ # QUARTILES
260
+ # ------------------------------------------------------
261
+
262
+ q1, median, q3 = values_df.approxQuantile(
263
+ self.column,
264
+ [0.25, 0.5, 0.75],
265
+ 0.01
266
+ )
267
+
268
+ # ------------------------------------------------------
269
+ # BARPLOT
270
+ # ------------------------------------------------------
271
+ #
272
+ # Only a bounded number of values is collected to the
273
+ # driver. This prevents high-cardinality numeric columns
274
+ # from generating very large Pandas DataFrames.
275
+ # ------------------------------------------------------
276
+
277
+ bar_counts = (
278
+ values_df
279
+ .groupBy(self.column)
280
+ .agg(
281
+ F.count("*").alias("frequency")
282
+ )
283
+ .orderBy(
284
+ F.desc("frequency"),
285
+ F.asc(self.column)
286
+ )
287
+ .limit(self.top_n_categories)
288
+ .toPandas()
289
+ )
290
+
291
+ # Sort selected values numerically for a more natural
292
+ # representation in the barplot.
293
+ bar_counts = bar_counts.sort_values(
294
+ self.column
295
+ )
296
+
297
+ # ------------------------------------------------------
298
+ # HISTOGRAM
299
+ # ------------------------------------------------------
300
+
301
+ if min_value == max_value:
302
+ histogram_df = (
303
+ values_df
304
+ .agg(
305
+ F.count("*").alias("frequency")
306
+ )
307
+ .withColumn(
308
+ "bin_center",
309
+ F.lit(float(min_value))
310
+ )
311
+ .select(
312
+ "bin_center",
313
+ "frequency"
314
+ )
315
+ )
316
+
317
+ bin_width = 1.0
318
+
319
+ else:
320
+ bin_width = (
321
+ (max_value - min_value)
322
+ / self.histogram_bins
323
+ )
324
+
325
+ bin_index = F.floor(
326
+ (F.col(self.column) - F.lit(min_value))
327
+ / F.lit(bin_width)
328
+ )
329
+
330
+ # The maximum value would otherwise fall into bin
331
+ # histogram_bins instead of histogram_bins - 1.
332
+ bin_index = F.when(
333
+ bin_index >= self.histogram_bins,
334
+ F.lit(self.histogram_bins - 1)
335
+ ).otherwise(bin_index)
336
+
337
+ histogram_df = (
338
+ values_df
339
+ .withColumn(
340
+ "_bin",
341
+ bin_index.cast("int")
342
+ )
343
+ .groupBy("_bin")
344
+ .agg(
345
+ F.count("*").alias("frequency")
346
+ )
347
+ .withColumn(
348
+ "bin_center",
349
+ F.lit(min_value)
350
+ + (
351
+ F.col("_bin") + F.lit(0.5)
352
+ ) * F.lit(bin_width)
353
+ )
354
+ .select(
355
+ "bin_center",
356
+ "frequency"
357
+ )
358
+ .orderBy("bin_center")
359
+ )
360
+
361
+ histogram = histogram_df.toPandas()
362
+
363
+ # ------------------------------------------------------
364
+ # FIGURE
365
+ # ------------------------------------------------------
366
+
367
+ fig = go.Figure()
368
+
369
+ fig.add_trace(
370
+ go.Bar(
371
+ x=bar_counts[self.column],
372
+ y=bar_counts["frequency"],
373
+ name="Barplot"
374
+ )
375
+ )
376
+
377
+ fig.add_trace(
378
+ go.Bar(
379
+ x=histogram["bin_center"],
380
+ y=histogram["frequency"],
381
+ width=bin_width,
382
+ name="Histogram",
383
+ visible=False
384
+ )
385
+ )
386
+
387
+ fig.add_trace(
388
+ go.Box(
389
+ q1=[q1],
390
+ median=[median],
391
+ mean=[mean_value],
392
+ q3=[q3],
393
+ lowerfence=[min_value],
394
+ upperfence=[max_value],
395
+ name="Boxplot",
396
+ visible=False,
397
+ boxpoints=False
398
+ )
399
+ )
400
+
401
+ fig.update_layout(
402
+ title=(
403
+ f"{self.column.title()} "
404
+ f"\t|\t(histogram_bins={self.histogram_bins})"
405
+ ),
406
+ xaxis_title=self.column,
407
+ yaxis_title="Count",
408
+ legend_title="Chart",
409
+ template="plotly_white"
410
+ )
411
+
412
+ self._add_toggle_menu(
413
+ fig,
414
+ ["Barplot", "Histogram", "Boxplot"]
415
+ )
416
+
417
+ return fig
418
+
419
+ def _build_categorical_figure(
420
+ self,
421
+ df: DataFrame
422
+ ) -> go.Figure:
423
+ """
424
+ Builds interactive visualizations for a categorical column.
425
+
426
+ Two visualization types are generated:
427
+
428
+ - Bar:
429
+ Displays the frequency of the most common categories.
430
+
431
+ - Treemap:
432
+ Displays the same category frequencies using a hierarchical
433
+ area-based representation.
434
+
435
+ Only the ``top_n_categories`` most frequent values are transferred
436
+ from Spark to the driver.
437
+
438
+ Parameters
439
+ ----------
440
+ df : pyspark.sql.DataFrame
441
+ Input Spark DataFrame containing the categorical column.
442
+
443
+ Returns
444
+ -------
445
+ plotly.graph_objects.Figure
446
+ Interactive Plotly figure containing the categorical
447
+ visualizations.
448
+
449
+ Raises
450
+ ------
451
+ ValueError
452
+ If the selected column contains no non-null values.
453
+
454
+ Notes
455
+ -----
456
+ Aggregation, ordering and limiting are performed by Spark before
457
+ conversion to Pandas. Therefore, the amount of data transferred
458
+ to the driver is bounded by ``top_n_categories``.
459
+ """
460
+ counts_df = (
461
+ df
462
+ .select(self.column)
463
+ .dropna()
464
+ .withColumn(
465
+ self.column,
466
+ F.col(self.column).cast("string")
467
+ )
468
+ .groupBy(self.column)
469
+ .agg(
470
+ F.count("*").alias("count")
471
+ )
472
+ .orderBy(
473
+ F.desc("count")
474
+ )
475
+ .limit(
476
+ self.top_n_categories
477
+ )
478
+ )
479
+
480
+ counts = counts_df.toPandas()
481
+
482
+ if counts.empty:
483
+ raise ValueError(
484
+ f"Column '{self.column}' contains no non-null values."
485
+ )
486
+
487
+ fig = go.Figure()
488
+
489
+ fig.add_trace(
490
+ go.Bar(
491
+ x=counts[self.column],
492
+ y=counts["count"],
493
+ name="Bar",
494
+ visible=True
495
+ )
496
+ )
497
+
498
+ fig.add_trace(
499
+ go.Treemap(
500
+ labels=counts[self.column],
501
+ parents=[""] * len(counts),
502
+ values=counts["count"],
503
+ name="Treemap",
504
+ visible=False
505
+ )
506
+ )
507
+
508
+ fig.update_layout(
509
+ title=(
510
+ f"{self.column.title()}"
511
+ f"\t|\t(top_n_categories="
512
+ f"{self.top_n_categories})"
513
+ ),
514
+ xaxis_title=self.column,
515
+ yaxis_title="Count",
516
+ legend_title="Chart",
517
+ template="plotly_white"
518
+ )
519
+
520
+ self._add_toggle_menu(
521
+ fig,
522
+ ["Bar", "Treemap"]
523
+ )
524
+
525
+ return fig
526
+
527
+ def _build_date_figure(
528
+ self,
529
+ df: DataFrame
530
+ ) -> go.Figure:
531
+ """
532
+ Builds interactive visualizations for a date column.
533
+
534
+ Three visualization types are generated:
535
+
536
+ - Year:
537
+ Displays the number of records for each year.
538
+
539
+ - Month:
540
+ Displays the number of records for each calendar month,
541
+ aggregated across all years.
542
+
543
+ - Sunburst:
544
+ Displays the hierarchical year-month distribution.
545
+
546
+ Parameters
547
+ ----------
548
+ df : pyspark.sql.DataFrame
549
+ Input Spark DataFrame containing the date column.
550
+
551
+ Returns
552
+ -------
553
+ plotly.graph_objects.Figure
554
+ Interactive Plotly figure containing the date
555
+ visualizations.
556
+
557
+ Raises
558
+ ------
559
+ ValueError
560
+ If the selected column contains no valid date values.
561
+
562
+ Notes
563
+ -----
564
+ Spark performs a single aggregation by year and month. The
565
+ resulting DataFrame contains at most one row for each observed
566
+ year-month combination and is therefore small enough to convert
567
+ to Pandas.
568
+
569
+ Year-level and month-level distributions are subsequently derived
570
+ locally from this already aggregated result, avoiding multiple
571
+ Spark groupBy operations over the original dataset.
572
+ """
573
+ date_df = (
574
+ df
575
+ .select(
576
+ F.to_date(
577
+ F.col(self.column)
578
+ ).alias(self.column)
579
+ )
580
+ .dropna()
581
+ .select(
582
+ F.year(
583
+ F.col(self.column)
584
+ ).alias("year"),
585
+ F.month(
586
+ F.col(self.column)
587
+ ).alias("month")
588
+ )
589
+ )
590
+
591
+ # ------------------------------------------------------
592
+ # SINGLE SPARK AGGREGATION
593
+ # ------------------------------------------------------
594
+
595
+ year_month_counts = (
596
+ date_df
597
+ .groupBy(
598
+ "year",
599
+ "month"
600
+ )
601
+ .agg(
602
+ F.count("*").alias("count")
603
+ )
604
+ .orderBy(
605
+ "year",
606
+ "month"
607
+ )
608
+ .toPandas()
609
+ )
610
+
611
+ if year_month_counts.empty:
612
+ raise ValueError(
613
+ f"Column '{self.column}' contains no valid date values."
614
+ )
615
+
616
+ # ------------------------------------------------------
617
+ # LOCAL AGGREGATIONS
618
+ # ------------------------------------------------------
619
+
620
+ year_counts = (
621
+ year_month_counts
622
+ .groupby(
623
+ "year",
624
+ as_index=False
625
+ )["count"]
626
+ .sum()
627
+ .sort_values("year")
628
+ )
629
+
630
+ month_counts = (
631
+ year_month_counts
632
+ .groupby(
633
+ "month",
634
+ as_index=False
635
+ )["count"]
636
+ .sum()
637
+ .sort_values("month")
638
+ )
639
+
640
+ month_nodes = year_month_counts.copy()
641
+
642
+ month_names = {
643
+ 1: "January",
644
+ 2: "February",
645
+ 3: "March",
646
+ 4: "April",
647
+ 5: "May",
648
+ 6: "June",
649
+ 7: "July",
650
+ 8: "August",
651
+ 9: "September",
652
+ 10: "October",
653
+ 11: "November",
654
+ 12: "December"
655
+ }
656
+
657
+ month_counts["month_name"] = (
658
+ month_counts["month"]
659
+ .map(month_names)
660
+ )
661
+
662
+ # ------------------------------------------------------
663
+ # SUNBURST NODES
664
+ # ------------------------------------------------------
665
+
666
+ year_nodes = year_counts.copy()
667
+
668
+ year_nodes["id"] = (
669
+ year_nodes["year"]
670
+ .astype(str)
671
+ )
672
+
673
+ year_nodes["label"] = (
674
+ year_nodes["year"]
675
+ .astype(str)
676
+ )
677
+
678
+ year_nodes["parent"] = ""
679
+
680
+ month_nodes["id"] = (
681
+ month_nodes["year"].astype(str)
682
+ + "-"
683
+ + month_nodes["month"]
684
+ .astype(str)
685
+ .str.zfill(2)
686
+ )
687
+
688
+ month_nodes["label"] = (
689
+ month_nodes["month"]
690
+ .map(month_names)
691
+ )
692
+
693
+ month_nodes["parent"] = (
694
+ month_nodes["year"]
695
+ .astype(str)
696
+ )
697
+
698
+ sunburst_ids = (
699
+ year_nodes["id"].tolist()
700
+ + month_nodes["id"].tolist()
701
+ )
702
+
703
+ sunburst_labels = (
704
+ year_nodes["label"].tolist()
705
+ + month_nodes["label"].tolist()
706
+ )
707
+
708
+ sunburst_parents = (
709
+ year_nodes["parent"].tolist()
710
+ + month_nodes["parent"].tolist()
711
+ )
712
+
713
+ sunburst_values = (
714
+ year_nodes["count"].tolist()
715
+ + month_nodes["count"].tolist()
716
+ )
717
+
718
+ # ------------------------------------------------------
719
+ # FIGURE
720
+ # ------------------------------------------------------
721
+
722
+ fig = go.Figure()
723
+
724
+ fig.add_trace(
725
+ go.Bar(
726
+ x=year_counts["year"].astype(str),
727
+ y=year_counts["count"],
728
+ name="Year",
729
+ visible=True
730
+ )
731
+ )
732
+
733
+ fig.add_trace(
734
+ go.Bar(
735
+ x=month_counts["month_name"],
736
+ y=month_counts["count"],
737
+ name="Month",
738
+ visible=False
739
+ )
740
+ )
741
+
742
+ fig.add_trace(
743
+ go.Sunburst(
744
+ ids=sunburst_ids,
745
+ labels=sunburst_labels,
746
+ parents=sunburst_parents,
747
+ values=sunburst_values,
748
+ branchvalues="total",
749
+ name="Sunburst",
750
+ visible=False,
751
+ sort=False
752
+ )
753
+ )
754
+
755
+ fig.update_layout(
756
+ title=f"{self.column.title()}",
757
+ xaxis_title="Date",
758
+ yaxis_title="Count",
759
+ legend_title="Chart",
760
+ template="plotly_white"
761
+ )
762
+
763
+ self._add_toggle_menu(
764
+ fig,
765
+ ["Year", "Month", "Sunburst"]
766
+ )
767
+
768
+ return fig
769
+
770
+ def transform(
771
+ self,
772
+ df: DataFrame
773
+ ) -> go.Figure:
774
+ """
775
+ Generates an interactive visualization for the configured column.
776
+
777
+ The semantic type of the selected column is inferred automatically
778
+ and the appropriate visualization strategy is applied.
779
+
780
+ Numeric columns are visualized using a barplot, histogram and
781
+ boxplot. Date columns are visualized using yearly, monthly and
782
+ hierarchical year-month distributions. Other columns are treated
783
+ as categorical and visualized using a bar chart and treemap.
784
+
785
+ Parameters
786
+ ----------
787
+ df : pyspark.sql.DataFrame
788
+ Input Spark DataFrame containing the column to visualize.
789
+
790
+ Returns
791
+ -------
792
+ plotly.graph_objects.Figure
793
+ Interactive Plotly figure corresponding to the inferred
794
+ semantic type of the selected column.
795
+
796
+ Raises
797
+ ------
798
+ ValueError
799
+ If the input DataFrame is None, is not Spark-compatible, or
800
+ does not contain the configured column.
801
+
802
+ Notes
803
+ -----
804
+ The method delegates the visualization process to one of the
805
+ specialized internal methods:
806
+
807
+ - ``_build_numeric_figure``
808
+ - ``_build_date_figure``
809
+ - ``_build_categorical_figure``
810
+
811
+ Data-intensive operations are performed with Spark before the
812
+ aggregated results are transferred to the driver for Plotly
813
+ visualization.
814
+ """
815
+ if df is None:
816
+ raise ValueError(
817
+ "Input DataFrame cannot be None."
818
+ )
819
+
820
+ # Compatible with standard Spark and Spark Connect.
821
+ if not hasattr(df, "select"):
822
+ raise ValueError(
823
+ "Input must be a Spark-compatible DataFrame."
824
+ )
825
+
826
+ if self.column not in df.columns:
827
+ raise ValueError(
828
+ f"Column '{self.column}' not found in DataFrame."
829
+ )
830
+
831
+ semantic_type = infer_semantic_type(
832
+ df,
833
+ self.column
834
+ )
835
+
836
+ if semantic_type == "numeric":
837
+ fig = self._build_numeric_figure(df)
838
+
839
+ elif semantic_type == "date":
840
+ fig = self._build_date_figure(df)
841
+
842
+ else:
843
+ fig = self._build_categorical_figure(df)
844
+
845
+ return fig