prepix 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
prepix/quality.py ADDED
@@ -0,0 +1,803 @@
1
+ """
2
+ Quality and EDA utilities for Prepix.
3
+
4
+ This module contains utilities for inspecting dataset quality,
5
+ including missing-value analysis, column summaries, duplicate detection,
6
+ uniqueness/cardinality analysis, and numerical feature summaries.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import warnings
11
+ from typing import Any, Iterable, List, Optional, Sequence, Union
12
+ import numpy as np
13
+ import pandas as pd
14
+
15
+
16
+ def _validate_dataframe(df: pd.DataFrame) -> None:
17
+ """Validate that the input is a pandas DataFrame."""
18
+ if not isinstance(df, pd.DataFrame):
19
+ raise TypeError("Input must be a pandas DataFrame.")
20
+
21
+
22
+ def _get_suggested_type(series: pd.Series, unique_count: Optional[int] = None) -> str:
23
+ """
24
+ Infer a high-level suggested data type for a column.
25
+
26
+ This is a heuristic classification, not a conversion.
27
+
28
+ Parameters
29
+ ----------
30
+ series : pandas.Series
31
+ The column series to inspect.
32
+ unique_count : int, optional
33
+ Precomputed unique value count to avoid redundant scans.
34
+ """
35
+ if pd.api.types.is_bool_dtype(series):
36
+ return "Binary"
37
+
38
+ if pd.api.types.is_numeric_dtype(series):
39
+ # Numeric columns remain numeric even if they contain
40
+ # only two unique values, such as 0 and 1.
41
+ return "Numeric"
42
+
43
+ if pd.api.types.is_datetime64_any_dtype(series):
44
+ return "Datetime"
45
+
46
+ if pd.api.types.is_timedelta64_dtype(series):
47
+ return "Timedelta"
48
+
49
+ if isinstance(series.dtype, pd.PeriodDtype):
50
+ return "Period"
51
+
52
+ if unique_count is None:
53
+ try:
54
+ unique_values = int(series.nunique(dropna=True))
55
+ except TypeError:
56
+ try:
57
+ unique_values = int(len(set(map(repr, series.dropna()))))
58
+ except Exception:
59
+ unique_values = -1
60
+ else:
61
+ unique_values = unique_count
62
+
63
+ if unique_values == 2:
64
+ return "Categorical (2 values)"
65
+
66
+ return "Categorical"
67
+
68
+
69
+ def missing_report(
70
+ df: pd.DataFrame,
71
+ low_threshold: float = 10.0,
72
+ medium_threshold: float = 30.0,
73
+ only_missing: bool = False,
74
+ ) -> pd.DataFrame:
75
+ """
76
+ Generate a missing-value report for a pandas DataFrame.
77
+
78
+ Parameters
79
+ ----------
80
+ df : pandas.DataFrame
81
+ Input DataFrame.
82
+
83
+ low_threshold : float, default=10.0
84
+ Maximum missing percentage considered "Low".
85
+
86
+ medium_threshold : float, default=30.0
87
+ Maximum missing percentage considered "Medium".
88
+ Values above this threshold are classified as "High".
89
+
90
+ only_missing : bool, default=False
91
+ If True, only columns with at least one missing value are included.
92
+
93
+ Returns
94
+ -------
95
+ pandas.DataFrame
96
+ Report containing:
97
+ - Column
98
+ - Missing Values
99
+ - Missing Percentage(%)
100
+ - Status
101
+ - Suggestion
102
+
103
+ Notes
104
+ -----
105
+ The suggestions are heuristic recommendations and should
106
+ not be treated as automatic decisions.
107
+
108
+ The input DataFrame is not modified.
109
+ """
110
+ _validate_dataframe(df)
111
+
112
+ if isinstance(low_threshold, bool) or not isinstance(low_threshold, (int, float)):
113
+ raise TypeError("low_threshold must be a numeric value.")
114
+ if np.isnan(low_threshold):
115
+ raise ValueError("low_threshold cannot be NaN.")
116
+
117
+ if isinstance(medium_threshold, bool) or not isinstance(medium_threshold, (int, float)):
118
+ raise TypeError("medium_threshold must be a numeric value.")
119
+ if np.isnan(medium_threshold):
120
+ raise ValueError("medium_threshold cannot be NaN.")
121
+
122
+ if not 0 <= low_threshold <= 100:
123
+ raise ValueError("low_threshold must be between 0 and 100.")
124
+
125
+ if not 0 <= medium_threshold <= 100:
126
+ raise ValueError("medium_threshold must be between 0 and 100.")
127
+
128
+ if low_threshold > medium_threshold:
129
+ raise ValueError(
130
+ "low_threshold cannot be greater than medium_threshold."
131
+ )
132
+
133
+ if not isinstance(only_missing, bool):
134
+ raise TypeError("only_missing must be a boolean.")
135
+
136
+ output_columns = [
137
+ "Column",
138
+ "Missing Values",
139
+ "Missing Percentage(%)",
140
+ "Status",
141
+ "Suggestion",
142
+ ]
143
+
144
+ if df.shape[1] == 0:
145
+ return pd.DataFrame(columns=output_columns)
146
+
147
+ n_rows = len(df)
148
+ missing_count = df.isna().sum()
149
+
150
+ if n_rows == 0:
151
+ missing_percentage = pd.Series(0.0, index=df.columns)
152
+ else:
153
+ missing_percentage = ((missing_count / n_rows) * 100).round(2)
154
+
155
+ def get_status(percent: float) -> str:
156
+ if percent == 0:
157
+ return "No Missing"
158
+ elif percent <= low_threshold:
159
+ return "Low"
160
+ elif percent <= medium_threshold:
161
+ return "Medium"
162
+ else:
163
+ return "High"
164
+
165
+ def get_suggestion(series: pd.Series, percent: float) -> str:
166
+ if percent == 0:
167
+ return "No Action Needed"
168
+
169
+ if percent > medium_threshold:
170
+ return "Review Before Imputation"
171
+
172
+ if pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(series):
173
+ return "Consider Median Imputation"
174
+
175
+ if (
176
+ pd.api.types.is_object_dtype(series)
177
+ or pd.api.types.is_string_dtype(series)
178
+ or isinstance(series.dtype, pd.CategoricalDtype)
179
+ or pd.api.types.is_bool_dtype(series)
180
+ ):
181
+ return "Consider Mode Imputation"
182
+
183
+ return "Review Column"
184
+
185
+ # Use positional indexing (iloc) to safely handle duplicate column names
186
+ suggestions = [
187
+ get_suggestion(df.iloc[:, i], missing_percentage.iloc[i])
188
+ for i in range(df.shape[1])
189
+ ]
190
+
191
+ report = pd.DataFrame(
192
+ {
193
+ "Column": list(df.columns),
194
+ "Missing Values": missing_count.to_numpy(),
195
+ "Missing Percentage(%)": missing_percentage.to_numpy(),
196
+ "Status": [get_status(p) for p in missing_percentage],
197
+ "Suggestion": suggestions,
198
+ }
199
+ )
200
+
201
+ if only_missing:
202
+ report = report[report["Missing Values"] > 0]
203
+
204
+ return (
205
+ report
206
+ .sort_values(
207
+ by="Missing Percentage(%)",
208
+ ascending=False,
209
+ kind="stable",
210
+ )
211
+ .reset_index(drop=True)
212
+ )
213
+
214
+
215
+ def column_summary(df: pd.DataFrame, dropna: bool = True) -> pd.DataFrame:
216
+ """
217
+ Generate a column-wise summary of a pandas DataFrame.
218
+
219
+ Parameters
220
+ ----------
221
+ df : pandas.DataFrame
222
+ Input DataFrame.
223
+
224
+ dropna : bool, default=True
225
+ Whether to exclude NaN/null values when counting unique values.
226
+ If True (default), matches pandas Series.nunique(dropna=True).
227
+ If False, NaN/null is counted as a distinct category.
228
+
229
+ Returns
230
+ -------
231
+ pandas.DataFrame
232
+ Summary containing:
233
+ - Column
234
+ - Data Type
235
+ - Missing Values
236
+ - Missing Percentage(%)
237
+ - Unique Values
238
+ - Memory(Bytes)
239
+ - Suggested Type
240
+
241
+ Notes
242
+ -----
243
+ The suggested type is a heuristic classification.
244
+ It does not modify the original DataFrame.
245
+ """
246
+ _validate_dataframe(df)
247
+
248
+ if not isinstance(dropna, bool):
249
+ raise TypeError("dropna must be a boolean.")
250
+
251
+ if df.shape[1] == 0:
252
+ return pd.DataFrame(
253
+ columns=[
254
+ "Column",
255
+ "Data Type",
256
+ "Missing Values",
257
+ "Missing Percentage(%)",
258
+ "Unique Values",
259
+ "Memory(Bytes)",
260
+ "Suggested Type",
261
+ ]
262
+ )
263
+
264
+ n_rows = len(df)
265
+ summary = []
266
+
267
+ for i in range(df.shape[1]):
268
+ column = df.columns[i]
269
+ series = df.iloc[:, i]
270
+
271
+ missing = int(series.isna().sum())
272
+ missing_percentage = (
273
+ round((missing / n_rows) * 100, 2) if n_rows > 0 else 0.0
274
+ )
275
+
276
+ try:
277
+ unique = int(series.nunique(dropna=dropna))
278
+ except TypeError:
279
+ try:
280
+ target = series if not dropna else series.dropna()
281
+ unique = int(len(set(map(repr, target))))
282
+ except Exception:
283
+ unique = -1
284
+
285
+ # deep=True gives a more useful estimate for object/string data.
286
+ memory = int(series.memory_usage(index=False, deep=True))
287
+
288
+ summary.append(
289
+ {
290
+ "Column": column,
291
+ "Data Type": str(series.dtype),
292
+ "Missing Values": missing,
293
+ "Missing Percentage(%)": missing_percentage,
294
+ "Unique Values": unique,
295
+ "Memory(Bytes)": memory,
296
+ "Suggested Type": _get_suggested_type(series, unique_count=unique),
297
+ }
298
+ )
299
+
300
+ return pd.DataFrame(summary)
301
+
302
+
303
+ def duplicate_report(
304
+ df: pd.DataFrame,
305
+ subset: Optional[Union[Sequence[str], str]] = None,
306
+ keep: Union[str, bool] = "first",
307
+ return_records: bool = False,
308
+ ) -> pd.DataFrame:
309
+ """
310
+ Generate a summary report of duplicate rows in a pandas DataFrame.
311
+
312
+ Parameters
313
+ ----------
314
+ df : pandas.DataFrame
315
+ Input DataFrame.
316
+
317
+ subset : sequence of column names or str, optional
318
+ Only consider certain columns for identifying duplicates.
319
+ If None, all columns are used.
320
+
321
+ keep : {'first', 'last', False}, default='first'
322
+ Determines which duplicates (if any) to mark:
323
+ - 'first': Mark duplicates as True except for the first occurrence.
324
+ - 'last': Mark duplicates as True except for the last occurrence.
325
+ - False: Mark all duplicates as True.
326
+
327
+ return_records : bool, default=False
328
+ If True, returns a DataFrame containing the duplicate records.
329
+ If False (default), returns the standard duplicate summary report.
330
+
331
+ Returns
332
+ -------
333
+ pandas.DataFrame
334
+ Report containing:
335
+ - Total Rows
336
+ - Duplicate Rows
337
+ - Duplicate Percentage(%)
338
+ - Unique Rows
339
+ - Status
340
+ - Suggestion
341
+
342
+ Notes
343
+ -----
344
+ A row is considered a duplicate if an identical row occurs earlier
345
+ in the DataFrame (following pandas `df.duplicated()` semantics).
346
+
347
+ The input DataFrame is not modified.
348
+ """
349
+ _validate_dataframe(df)
350
+
351
+ if not isinstance(return_records, bool):
352
+ raise TypeError("return_records must be a boolean.")
353
+
354
+ if keep not in ("first", "last", False):
355
+ raise ValueError("keep must be one of 'first', 'last', or False.")
356
+
357
+ if subset is not None:
358
+ if isinstance(subset, str):
359
+ subset_cols = [subset]
360
+ elif hasattr(subset, "__iter__"):
361
+ subset_cols = list(subset)
362
+ else:
363
+ raise TypeError("subset must be a string, list of strings, or None.")
364
+
365
+ missing_cols = [col for col in subset_cols if col not in df.columns]
366
+ if missing_cols:
367
+ raise KeyError(f"Columns not found in DataFrame: {missing_cols}")
368
+ else:
369
+ subset_cols = None
370
+
371
+ total_rows = len(df)
372
+ if total_rows == 0:
373
+ duplicate_rows = 0
374
+ duplicate_percentage = 0.0
375
+ unique_rows = 0
376
+ status = "No Duplicates"
377
+ suggestion = "No action required"
378
+ dup_mask = pd.Series([], dtype=bool)
379
+ else:
380
+ try:
381
+ dup_mask = df.duplicated(subset=subset_cols, keep=keep)
382
+ duplicate_rows = int(dup_mask.sum())
383
+ except TypeError:
384
+ try:
385
+ # Map unhashable objects (lists, dicts, sets) to their string representations
386
+ map_fn = getattr(df, "map", getattr(df, "applymap", None))
387
+ if map_fn is not None:
388
+ safe_df = map_fn(lambda x: repr(x) if isinstance(x, (list, dict, set)) else x)
389
+ dup_mask = safe_df.duplicated(subset=subset_cols, keep=keep)
390
+ duplicate_rows = int(dup_mask.sum())
391
+ else:
392
+ dup_mask = pd.Series([False] * total_rows, index=df.index)
393
+ duplicate_rows = 0
394
+ except Exception:
395
+ dup_mask = pd.Series([False] * total_rows, index=df.index)
396
+ duplicate_rows = 0
397
+
398
+ duplicate_percentage = round((duplicate_rows / total_rows) * 100, 2)
399
+ unique_rows = total_rows - duplicate_rows
400
+
401
+ if duplicate_rows == 0:
402
+ status = "No Duplicates"
403
+ suggestion = "No action required"
404
+ else:
405
+ status = "Duplicates Found"
406
+ suggestion = "Consider reviewing and removing duplicate records"
407
+
408
+ if return_records:
409
+ if total_rows == 0:
410
+ return df.iloc[0:0].copy()
411
+ return df.iloc[dup_mask.to_numpy()].copy()
412
+
413
+ return pd.DataFrame(
414
+ [
415
+ {
416
+ "Total Rows": total_rows,
417
+ "Duplicate Rows": duplicate_rows,
418
+ "Duplicate Percentage(%)": duplicate_percentage,
419
+ "Unique Rows": unique_rows,
420
+ "Status": status,
421
+ "Suggestion": suggestion,
422
+ }
423
+ ]
424
+ )
425
+
426
+
427
+ def unique_report(df: pd.DataFrame, dropna: bool = False) -> pd.DataFrame:
428
+ """
429
+ Generate a uniqueness and cardinality report for every column in a DataFrame.
430
+
431
+ Parameters
432
+ ----------
433
+ df : pandas.DataFrame
434
+ Input DataFrame.
435
+
436
+ dropna : bool, default=False
437
+ Whether to exclude NaN/null values when counting unique values.
438
+ If False (default), NaN/null is counted as a distinct category if present.
439
+
440
+ Returns
441
+ -------
442
+ pandas.DataFrame
443
+ Report containing:
444
+ - Column
445
+ - Data Type
446
+ - Total Values
447
+ - Unique Values
448
+ - Unique Percentage(%)
449
+ - Duplicate Values
450
+ - Cardinality
451
+ - Status
452
+ - Suggestion
453
+
454
+ Notes
455
+ -----
456
+ Cardinality classifications:
457
+ - Constant: Exactly 1 unique value (Zero variance).
458
+ - Very High (Unique): 100% unique values (Potential unique identifier).
459
+ - Very High: >= 90% unique values (Near-unique / high diversity).
460
+ - High: 50% - 90% unique values.
461
+ - Medium: 10% - 50% unique values.
462
+ - Low: < 10% unique values.
463
+
464
+ The suggestions are heuristic recommendations and do not modify the input DataFrame.
465
+ """
466
+ _validate_dataframe(df)
467
+
468
+ if not isinstance(dropna, bool):
469
+ raise TypeError("dropna must be a boolean.")
470
+
471
+ if df.shape[1] == 0:
472
+ return pd.DataFrame(
473
+ columns=[
474
+ "Column",
475
+ "Data Type",
476
+ "Total Values",
477
+ "Unique Values",
478
+ "Unique Percentage(%)",
479
+ "Duplicate Values",
480
+ "Cardinality",
481
+ "Status",
482
+ "Suggestion",
483
+ ]
484
+ )
485
+
486
+ n_rows = len(df)
487
+ report_rows = []
488
+
489
+ for i in range(df.shape[1]):
490
+ column = df.columns[i]
491
+ series = df.iloc[:, i]
492
+ dtype_str = str(series.dtype)
493
+
494
+ if n_rows == 0:
495
+ report_rows.append(
496
+ {
497
+ "Column": column,
498
+ "Data Type": dtype_str,
499
+ "Total Values": 0,
500
+ "Unique Values": 0,
501
+ "Unique Percentage(%)": 0.0,
502
+ "Duplicate Values": 0,
503
+ "Cardinality": "Empty",
504
+ "Status": "Empty Column",
505
+ "Suggestion": "No action needed",
506
+ }
507
+ )
508
+ continue
509
+
510
+ try:
511
+ unique_count = int(series.nunique(dropna=dropna))
512
+ except TypeError:
513
+ try:
514
+ target = series if not dropna else series.dropna()
515
+ unique_count = int(len(set(map(repr, target))))
516
+ except Exception:
517
+ unique_count = n_rows
518
+ unique_pct = round((unique_count / n_rows) * 100, 2)
519
+ valid_count = int(series.count())
520
+ duplicate_count = (n_rows - unique_count) if not dropna else max(0, valid_count - unique_count)
521
+
522
+ col_str = str(column).lower().strip()
523
+ is_id_name = any(
524
+ x in col_str.replace("-", "_").split("_")
525
+ for x in ["id", "key", "code", "num", "number"]
526
+ ) or col_str in ["id", "key", "code", "uuid", "guid"]
527
+
528
+ if unique_count == 0:
529
+ cardinality = "Empty"
530
+ status = "All Missing"
531
+ suggestion = "Column contains only missing values; consider dropping or imputing"
532
+ elif unique_count == 1:
533
+ cardinality = "Constant"
534
+ status = "Zero Variance"
535
+ suggestion = "Consider reviewing column; constant value across all rows"
536
+ elif unique_count == n_rows:
537
+ cardinality = "Very High (Unique)"
538
+ status = "Potential Identifier"
539
+ if pd.api.types.is_float_dtype(series):
540
+ suggestion = "Continuous numeric distribution with high uniqueness; review if feature or identifier"
541
+ else:
542
+ suggestion = "Review whether this column is a unique identifier"
543
+ elif unique_pct >= 90.0:
544
+ cardinality = "Very High"
545
+ status = "Near-Unique"
546
+ if is_id_name or (pd.api.types.is_integer_dtype(series) and not pd.api.types.is_bool_dtype(series)):
547
+ suggestion = "Review whether this column is an identifier, sequence, or high-cardinality feature"
548
+ elif pd.api.types.is_float_dtype(series):
549
+ suggestion = "Continuous numeric distribution with high uniqueness"
550
+ else:
551
+ suggestion = "Review whether this column is an identifier or high-cardinality feature"
552
+ elif unique_pct >= 50.0:
553
+ cardinality = "High"
554
+ status = "High Diversity"
555
+ if is_id_name:
556
+ suggestion = "High uniqueness discrete values; review whether this column is an identifier or key"
557
+ elif pd.api.types.is_float_dtype(series):
558
+ suggestion = "High uniqueness numeric distribution; review distribution spread"
559
+ elif pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(series):
560
+ suggestion = "High uniqueness discrete numeric values; review distribution"
561
+ else:
562
+ suggestion = "High cardinality categorical; review before one-hot encoding or grouping"
563
+ elif unique_pct >= 10.0:
564
+ cardinality = "Medium"
565
+ status = "Moderate Diversity"
566
+ suggestion = "Suitable for standard encoding or analysis"
567
+ else:
568
+ cardinality = "Low"
569
+ if unique_count == 2:
570
+ status = "Binary"
571
+ suggestion = "Suitable for binary encoding"
572
+ else:
573
+ status = "Low Diversity"
574
+ suggestion = "Suitable for categorical encoding"
575
+
576
+ report_rows.append(
577
+ {
578
+ "Column": column,
579
+ "Data Type": dtype_str,
580
+ "Total Values": n_rows,
581
+ "Unique Values": unique_count,
582
+ "Unique Percentage(%)": unique_pct,
583
+ "Duplicate Values": duplicate_count,
584
+ "Cardinality": cardinality,
585
+ "Status": status,
586
+ "Suggestion": suggestion,
587
+ }
588
+ )
589
+
590
+ return pd.DataFrame(report_rows)
591
+
592
+
593
+ def numeric_summary(
594
+ df: pd.DataFrame,
595
+ extended: bool = False,
596
+ percentiles: Optional[Sequence[float]] = None,
597
+ ) -> pd.DataFrame:
598
+ """
599
+ Generate a statistical summary for all numerical columns in a DataFrame.
600
+
601
+ Parameters
602
+ ----------
603
+ df : pandas.DataFrame
604
+ Input DataFrame.
605
+
606
+ extended : bool, default=False
607
+ Whether to include extended statistical diagnostics:
608
+ - IQR (Interquartile Range)
609
+ - Skewness (Fisher-Pearson distribution asymmetry)
610
+ - Kurtosis (Fisher excess peakedness)
611
+ - Zeros (Count of zero values)
612
+ - Negatives (Count of negative values)
613
+ - Outliers (Count of observations beyond 1.5 * IQR)
614
+
615
+ percentiles : sequence of float, optional
616
+ Additional percentiles to calculate, e.g. [0.05, 0.95].
617
+ Values must be between 0.0 and 1.0.
618
+
619
+ Returns
620
+ -------
621
+ pandas.DataFrame
622
+ Summary report containing one row per numerical column:
623
+ - Column: Name of the numerical column
624
+ - Data Type: String representation of the column's data type
625
+ - Count: Number of non-null (valid) numerical observations
626
+ - Missing Values: Number of missing/null (NaN) entries
627
+ - Mean: Arithmetic average of valid observations
628
+ - Median: 50th percentile of valid observations
629
+ - Std: Sample standard deviation (ddof=1)
630
+ - Min: Minimum valid numerical value
631
+ - Q1: 25th percentile of valid observations
632
+ - Q3: 75th percentile of valid observations
633
+ - Max: Maximum valid numerical value
634
+ - Range: Spread between Maximum and Minimum (Max - Min)
635
+ [If extended=True: IQR, Skewness, Kurtosis, Zeros, Negatives, Outliers]
636
+ [If percentiles: Corresponding percentile columns, e.g. P5, P95]
637
+
638
+ Notes
639
+ -----
640
+ - Numerical Column Detection: Includes standard integer, float, nullable integer,
641
+ and nullable float dtypes. Boolean and complex columns are treated as non-numerical
642
+ and excluded from the summary.
643
+ - Missing Values: Statistics are calculated ignoring NaN values. Columns with 0 valid
644
+ observations return NaN for statistical measures without crashing.
645
+ - Standard Deviation: Uses sample standard deviation with N - 1 degrees of freedom (ddof=1).
646
+ Columns with single valid observations return NaN for Std.
647
+ - Infinity: Infinite values (np.inf, -np.inf) are retained according to standard
648
+ Pandas semantics and not converted to NaN.
649
+ - Outlier Detection: Identifies observations falling outside [Q1 - 1.5*IQR, Q3 + 1.5*IQR].
650
+ - The input DataFrame is not modified.
651
+ """
652
+ _validate_dataframe(df)
653
+
654
+ if not isinstance(extended, bool):
655
+ raise TypeError("extended must be a boolean.")
656
+
657
+ validated_percentiles: List[float] = []
658
+ if percentiles is not None:
659
+ if not hasattr(percentiles, "__iter__") or isinstance(percentiles, (str, bytes)):
660
+ raise TypeError("percentiles must be a sequence of floats between 0 and 1.")
661
+ for p in percentiles:
662
+ if not isinstance(p, (int, float)) or isinstance(p, bool) or not (0.0 <= p <= 1.0):
663
+ raise ValueError(f"Percentile {p} must be a number between 0 and 1.")
664
+ validated_percentiles.append(float(p))
665
+
666
+ base_columns = [
667
+ "Column",
668
+ "Data Type",
669
+ "Count",
670
+ "Missing Values",
671
+ "Mean",
672
+ "Median",
673
+ "Std",
674
+ "Min",
675
+ "Q1",
676
+ "Q3",
677
+ "Max",
678
+ "Range",
679
+ ]
680
+
681
+ output_columns = list(base_columns)
682
+ if extended:
683
+ output_columns.extend(["IQR", "Skewness", "Kurtosis", "Zeros", "Negatives", "Outliers"])
684
+ for p in validated_percentiles:
685
+ p_pct = p * 100
686
+ p_col = f"P{int(p_pct)}" if p_pct.is_integer() else f"P{p_pct:.1f}"
687
+ if p_col not in output_columns:
688
+ output_columns.append(p_col)
689
+
690
+ if df.shape[1] == 0:
691
+ return pd.DataFrame(columns=output_columns)
692
+
693
+ report_rows = []
694
+
695
+ for i in range(df.shape[1]):
696
+ column = df.columns[i]
697
+ series = df.iloc[:, i]
698
+
699
+ # Numerical detection: integer/float (including nullable), excluding bool and complex
700
+ if not pd.api.types.is_numeric_dtype(series) or pd.api.types.is_bool_dtype(series) or pd.api.types.is_complex_dtype(series):
701
+ continue
702
+
703
+ valid_count = int(series.count())
704
+ missing_count = int(series.isna().sum())
705
+
706
+ with warnings.catch_warnings():
707
+ warnings.simplefilter("ignore", category=RuntimeWarning)
708
+
709
+ if valid_count == 0:
710
+ mean_val = np.nan
711
+ median_val = np.nan
712
+ std_val = np.nan
713
+ min_val = np.nan
714
+ q1_val = np.nan
715
+ q3_val = np.nan
716
+ max_val = np.nan
717
+ range_val = np.nan
718
+ iqr_val = np.nan
719
+ skew_val = np.nan
720
+ kurt_val = np.nan
721
+ zeros_val = 0
722
+ negatives_val = 0
723
+ outliers_val = 0
724
+ else:
725
+ mean_val = series.mean()
726
+ median_val = series.median()
727
+ std_val = series.std() if valid_count > 1 else np.nan
728
+ min_val = series.min()
729
+ q1_val = series.quantile(0.25)
730
+ q3_val = series.quantile(0.75)
731
+ max_val = series.max()
732
+ try:
733
+ range_val = max_val - min_val
734
+ except Exception:
735
+ range_val = np.nan
736
+
737
+ if extended:
738
+ try:
739
+ iqr_val = q3_val - q1_val if (pd.notna(q3_val) and pd.notna(q1_val)) else np.nan
740
+ except Exception:
741
+ iqr_val = np.nan
742
+
743
+ skew_val = float(series.skew()) if valid_count >= 3 else np.nan
744
+ kurt_val = float(series.kurt()) if valid_count >= 4 else np.nan
745
+
746
+ try:
747
+ zeros_val = int((series == 0).sum())
748
+ except Exception:
749
+ zeros_val = 0
750
+
751
+ try:
752
+ negatives_val = int((series < 0).sum())
753
+ except Exception:
754
+ negatives_val = 0
755
+
756
+ if pd.notna(iqr_val) and not np.isinf(iqr_val) and iqr_val >= 0:
757
+ lower_bound = q1_val - 1.5 * iqr_val
758
+ upper_bound = q3_val + 1.5 * iqr_val
759
+ try:
760
+ outliers_val = int(((series < lower_bound) | (series > upper_bound)).sum())
761
+ except Exception:
762
+ outliers_val = 0
763
+ else:
764
+ outliers_val = 0
765
+
766
+ row_dict: dict[str, Any] = {
767
+ "Column": column,
768
+ "Data Type": str(series.dtype),
769
+ "Count": valid_count,
770
+ "Missing Values": missing_count,
771
+ "Mean": mean_val,
772
+ "Median": median_val,
773
+ "Std": std_val,
774
+ "Min": min_val,
775
+ "Q1": q1_val,
776
+ "Q3": q3_val,
777
+ "Max": max_val,
778
+ "Range": range_val,
779
+ }
780
+
781
+ if extended:
782
+ row_dict["IQR"] = iqr_val
783
+ row_dict["Skewness"] = skew_val
784
+ row_dict["Kurtosis"] = kurt_val
785
+ row_dict["Zeros"] = zeros_val
786
+ row_dict["Negatives"] = negatives_val
787
+ row_dict["Outliers"] = outliers_val
788
+
789
+ for p in validated_percentiles:
790
+ p_pct = p * 100
791
+ p_col = f"P{int(p_pct)}" if p_pct.is_integer() else f"P{p_pct:.1f}"
792
+ row_dict[p_col] = series.quantile(p) if valid_count > 0 else np.nan
793
+
794
+ report_rows.append(row_dict)
795
+
796
+ if not report_rows:
797
+ return pd.DataFrame(columns=output_columns)
798
+
799
+ return pd.DataFrame(report_rows)
800
+
801
+
802
+ # Convenience imports for cross-module accessibility
803
+ from .report import PrepixReport, report # noqa: E402, F401