prepix 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prepix/__init__.py +22 -0
- prepix/quality.py +803 -0
- prepix/report.py +412 -0
- prepix/schema.py +496 -0
- prepix-0.1.0.dist-info/METADATA +255 -0
- prepix-0.1.0.dist-info/RECORD +9 -0
- prepix-0.1.0.dist-info/WHEEL +5 -0
- prepix-0.1.0.dist-info/licenses/LICENSE +21 -0
- prepix-0.1.0.dist-info/top_level.txt +1 -0
prepix/quality.py
ADDED
|
@@ -0,0 +1,803 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Quality and EDA utilities for Prepix.
|
|
3
|
+
|
|
4
|
+
This module contains utilities for inspecting dataset quality,
|
|
5
|
+
including missing-value analysis, column summaries, duplicate detection,
|
|
6
|
+
uniqueness/cardinality analysis, and numerical feature summaries.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import warnings
|
|
11
|
+
from typing import Any, Iterable, List, Optional, Sequence, Union
|
|
12
|
+
import numpy as np
|
|
13
|
+
import pandas as pd
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _validate_dataframe(df: pd.DataFrame) -> None:
|
|
17
|
+
"""Validate that the input is a pandas DataFrame."""
|
|
18
|
+
if not isinstance(df, pd.DataFrame):
|
|
19
|
+
raise TypeError("Input must be a pandas DataFrame.")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _get_suggested_type(series: pd.Series, unique_count: Optional[int] = None) -> str:
|
|
23
|
+
"""
|
|
24
|
+
Infer a high-level suggested data type for a column.
|
|
25
|
+
|
|
26
|
+
This is a heuristic classification, not a conversion.
|
|
27
|
+
|
|
28
|
+
Parameters
|
|
29
|
+
----------
|
|
30
|
+
series : pandas.Series
|
|
31
|
+
The column series to inspect.
|
|
32
|
+
unique_count : int, optional
|
|
33
|
+
Precomputed unique value count to avoid redundant scans.
|
|
34
|
+
"""
|
|
35
|
+
if pd.api.types.is_bool_dtype(series):
|
|
36
|
+
return "Binary"
|
|
37
|
+
|
|
38
|
+
if pd.api.types.is_numeric_dtype(series):
|
|
39
|
+
# Numeric columns remain numeric even if they contain
|
|
40
|
+
# only two unique values, such as 0 and 1.
|
|
41
|
+
return "Numeric"
|
|
42
|
+
|
|
43
|
+
if pd.api.types.is_datetime64_any_dtype(series):
|
|
44
|
+
return "Datetime"
|
|
45
|
+
|
|
46
|
+
if pd.api.types.is_timedelta64_dtype(series):
|
|
47
|
+
return "Timedelta"
|
|
48
|
+
|
|
49
|
+
if isinstance(series.dtype, pd.PeriodDtype):
|
|
50
|
+
return "Period"
|
|
51
|
+
|
|
52
|
+
if unique_count is None:
|
|
53
|
+
try:
|
|
54
|
+
unique_values = int(series.nunique(dropna=True))
|
|
55
|
+
except TypeError:
|
|
56
|
+
try:
|
|
57
|
+
unique_values = int(len(set(map(repr, series.dropna()))))
|
|
58
|
+
except Exception:
|
|
59
|
+
unique_values = -1
|
|
60
|
+
else:
|
|
61
|
+
unique_values = unique_count
|
|
62
|
+
|
|
63
|
+
if unique_values == 2:
|
|
64
|
+
return "Categorical (2 values)"
|
|
65
|
+
|
|
66
|
+
return "Categorical"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def missing_report(
|
|
70
|
+
df: pd.DataFrame,
|
|
71
|
+
low_threshold: float = 10.0,
|
|
72
|
+
medium_threshold: float = 30.0,
|
|
73
|
+
only_missing: bool = False,
|
|
74
|
+
) -> pd.DataFrame:
|
|
75
|
+
"""
|
|
76
|
+
Generate a missing-value report for a pandas DataFrame.
|
|
77
|
+
|
|
78
|
+
Parameters
|
|
79
|
+
----------
|
|
80
|
+
df : pandas.DataFrame
|
|
81
|
+
Input DataFrame.
|
|
82
|
+
|
|
83
|
+
low_threshold : float, default=10.0
|
|
84
|
+
Maximum missing percentage considered "Low".
|
|
85
|
+
|
|
86
|
+
medium_threshold : float, default=30.0
|
|
87
|
+
Maximum missing percentage considered "Medium".
|
|
88
|
+
Values above this threshold are classified as "High".
|
|
89
|
+
|
|
90
|
+
only_missing : bool, default=False
|
|
91
|
+
If True, only columns with at least one missing value are included.
|
|
92
|
+
|
|
93
|
+
Returns
|
|
94
|
+
-------
|
|
95
|
+
pandas.DataFrame
|
|
96
|
+
Report containing:
|
|
97
|
+
- Column
|
|
98
|
+
- Missing Values
|
|
99
|
+
- Missing Percentage(%)
|
|
100
|
+
- Status
|
|
101
|
+
- Suggestion
|
|
102
|
+
|
|
103
|
+
Notes
|
|
104
|
+
-----
|
|
105
|
+
The suggestions are heuristic recommendations and should
|
|
106
|
+
not be treated as automatic decisions.
|
|
107
|
+
|
|
108
|
+
The input DataFrame is not modified.
|
|
109
|
+
"""
|
|
110
|
+
_validate_dataframe(df)
|
|
111
|
+
|
|
112
|
+
if isinstance(low_threshold, bool) or not isinstance(low_threshold, (int, float)):
|
|
113
|
+
raise TypeError("low_threshold must be a numeric value.")
|
|
114
|
+
if np.isnan(low_threshold):
|
|
115
|
+
raise ValueError("low_threshold cannot be NaN.")
|
|
116
|
+
|
|
117
|
+
if isinstance(medium_threshold, bool) or not isinstance(medium_threshold, (int, float)):
|
|
118
|
+
raise TypeError("medium_threshold must be a numeric value.")
|
|
119
|
+
if np.isnan(medium_threshold):
|
|
120
|
+
raise ValueError("medium_threshold cannot be NaN.")
|
|
121
|
+
|
|
122
|
+
if not 0 <= low_threshold <= 100:
|
|
123
|
+
raise ValueError("low_threshold must be between 0 and 100.")
|
|
124
|
+
|
|
125
|
+
if not 0 <= medium_threshold <= 100:
|
|
126
|
+
raise ValueError("medium_threshold must be between 0 and 100.")
|
|
127
|
+
|
|
128
|
+
if low_threshold > medium_threshold:
|
|
129
|
+
raise ValueError(
|
|
130
|
+
"low_threshold cannot be greater than medium_threshold."
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
if not isinstance(only_missing, bool):
|
|
134
|
+
raise TypeError("only_missing must be a boolean.")
|
|
135
|
+
|
|
136
|
+
output_columns = [
|
|
137
|
+
"Column",
|
|
138
|
+
"Missing Values",
|
|
139
|
+
"Missing Percentage(%)",
|
|
140
|
+
"Status",
|
|
141
|
+
"Suggestion",
|
|
142
|
+
]
|
|
143
|
+
|
|
144
|
+
if df.shape[1] == 0:
|
|
145
|
+
return pd.DataFrame(columns=output_columns)
|
|
146
|
+
|
|
147
|
+
n_rows = len(df)
|
|
148
|
+
missing_count = df.isna().sum()
|
|
149
|
+
|
|
150
|
+
if n_rows == 0:
|
|
151
|
+
missing_percentage = pd.Series(0.0, index=df.columns)
|
|
152
|
+
else:
|
|
153
|
+
missing_percentage = ((missing_count / n_rows) * 100).round(2)
|
|
154
|
+
|
|
155
|
+
def get_status(percent: float) -> str:
|
|
156
|
+
if percent == 0:
|
|
157
|
+
return "No Missing"
|
|
158
|
+
elif percent <= low_threshold:
|
|
159
|
+
return "Low"
|
|
160
|
+
elif percent <= medium_threshold:
|
|
161
|
+
return "Medium"
|
|
162
|
+
else:
|
|
163
|
+
return "High"
|
|
164
|
+
|
|
165
|
+
def get_suggestion(series: pd.Series, percent: float) -> str:
|
|
166
|
+
if percent == 0:
|
|
167
|
+
return "No Action Needed"
|
|
168
|
+
|
|
169
|
+
if percent > medium_threshold:
|
|
170
|
+
return "Review Before Imputation"
|
|
171
|
+
|
|
172
|
+
if pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(series):
|
|
173
|
+
return "Consider Median Imputation"
|
|
174
|
+
|
|
175
|
+
if (
|
|
176
|
+
pd.api.types.is_object_dtype(series)
|
|
177
|
+
or pd.api.types.is_string_dtype(series)
|
|
178
|
+
or isinstance(series.dtype, pd.CategoricalDtype)
|
|
179
|
+
or pd.api.types.is_bool_dtype(series)
|
|
180
|
+
):
|
|
181
|
+
return "Consider Mode Imputation"
|
|
182
|
+
|
|
183
|
+
return "Review Column"
|
|
184
|
+
|
|
185
|
+
# Use positional indexing (iloc) to safely handle duplicate column names
|
|
186
|
+
suggestions = [
|
|
187
|
+
get_suggestion(df.iloc[:, i], missing_percentage.iloc[i])
|
|
188
|
+
for i in range(df.shape[1])
|
|
189
|
+
]
|
|
190
|
+
|
|
191
|
+
report = pd.DataFrame(
|
|
192
|
+
{
|
|
193
|
+
"Column": list(df.columns),
|
|
194
|
+
"Missing Values": missing_count.to_numpy(),
|
|
195
|
+
"Missing Percentage(%)": missing_percentage.to_numpy(),
|
|
196
|
+
"Status": [get_status(p) for p in missing_percentage],
|
|
197
|
+
"Suggestion": suggestions,
|
|
198
|
+
}
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
if only_missing:
|
|
202
|
+
report = report[report["Missing Values"] > 0]
|
|
203
|
+
|
|
204
|
+
return (
|
|
205
|
+
report
|
|
206
|
+
.sort_values(
|
|
207
|
+
by="Missing Percentage(%)",
|
|
208
|
+
ascending=False,
|
|
209
|
+
kind="stable",
|
|
210
|
+
)
|
|
211
|
+
.reset_index(drop=True)
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def column_summary(df: pd.DataFrame, dropna: bool = True) -> pd.DataFrame:
|
|
216
|
+
"""
|
|
217
|
+
Generate a column-wise summary of a pandas DataFrame.
|
|
218
|
+
|
|
219
|
+
Parameters
|
|
220
|
+
----------
|
|
221
|
+
df : pandas.DataFrame
|
|
222
|
+
Input DataFrame.
|
|
223
|
+
|
|
224
|
+
dropna : bool, default=True
|
|
225
|
+
Whether to exclude NaN/null values when counting unique values.
|
|
226
|
+
If True (default), matches pandas Series.nunique(dropna=True).
|
|
227
|
+
If False, NaN/null is counted as a distinct category.
|
|
228
|
+
|
|
229
|
+
Returns
|
|
230
|
+
-------
|
|
231
|
+
pandas.DataFrame
|
|
232
|
+
Summary containing:
|
|
233
|
+
- Column
|
|
234
|
+
- Data Type
|
|
235
|
+
- Missing Values
|
|
236
|
+
- Missing Percentage(%)
|
|
237
|
+
- Unique Values
|
|
238
|
+
- Memory(Bytes)
|
|
239
|
+
- Suggested Type
|
|
240
|
+
|
|
241
|
+
Notes
|
|
242
|
+
-----
|
|
243
|
+
The suggested type is a heuristic classification.
|
|
244
|
+
It does not modify the original DataFrame.
|
|
245
|
+
"""
|
|
246
|
+
_validate_dataframe(df)
|
|
247
|
+
|
|
248
|
+
if not isinstance(dropna, bool):
|
|
249
|
+
raise TypeError("dropna must be a boolean.")
|
|
250
|
+
|
|
251
|
+
if df.shape[1] == 0:
|
|
252
|
+
return pd.DataFrame(
|
|
253
|
+
columns=[
|
|
254
|
+
"Column",
|
|
255
|
+
"Data Type",
|
|
256
|
+
"Missing Values",
|
|
257
|
+
"Missing Percentage(%)",
|
|
258
|
+
"Unique Values",
|
|
259
|
+
"Memory(Bytes)",
|
|
260
|
+
"Suggested Type",
|
|
261
|
+
]
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
n_rows = len(df)
|
|
265
|
+
summary = []
|
|
266
|
+
|
|
267
|
+
for i in range(df.shape[1]):
|
|
268
|
+
column = df.columns[i]
|
|
269
|
+
series = df.iloc[:, i]
|
|
270
|
+
|
|
271
|
+
missing = int(series.isna().sum())
|
|
272
|
+
missing_percentage = (
|
|
273
|
+
round((missing / n_rows) * 100, 2) if n_rows > 0 else 0.0
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
try:
|
|
277
|
+
unique = int(series.nunique(dropna=dropna))
|
|
278
|
+
except TypeError:
|
|
279
|
+
try:
|
|
280
|
+
target = series if not dropna else series.dropna()
|
|
281
|
+
unique = int(len(set(map(repr, target))))
|
|
282
|
+
except Exception:
|
|
283
|
+
unique = -1
|
|
284
|
+
|
|
285
|
+
# deep=True gives a more useful estimate for object/string data.
|
|
286
|
+
memory = int(series.memory_usage(index=False, deep=True))
|
|
287
|
+
|
|
288
|
+
summary.append(
|
|
289
|
+
{
|
|
290
|
+
"Column": column,
|
|
291
|
+
"Data Type": str(series.dtype),
|
|
292
|
+
"Missing Values": missing,
|
|
293
|
+
"Missing Percentage(%)": missing_percentage,
|
|
294
|
+
"Unique Values": unique,
|
|
295
|
+
"Memory(Bytes)": memory,
|
|
296
|
+
"Suggested Type": _get_suggested_type(series, unique_count=unique),
|
|
297
|
+
}
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
return pd.DataFrame(summary)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def duplicate_report(
|
|
304
|
+
df: pd.DataFrame,
|
|
305
|
+
subset: Optional[Union[Sequence[str], str]] = None,
|
|
306
|
+
keep: Union[str, bool] = "first",
|
|
307
|
+
return_records: bool = False,
|
|
308
|
+
) -> pd.DataFrame:
|
|
309
|
+
"""
|
|
310
|
+
Generate a summary report of duplicate rows in a pandas DataFrame.
|
|
311
|
+
|
|
312
|
+
Parameters
|
|
313
|
+
----------
|
|
314
|
+
df : pandas.DataFrame
|
|
315
|
+
Input DataFrame.
|
|
316
|
+
|
|
317
|
+
subset : sequence of column names or str, optional
|
|
318
|
+
Only consider certain columns for identifying duplicates.
|
|
319
|
+
If None, all columns are used.
|
|
320
|
+
|
|
321
|
+
keep : {'first', 'last', False}, default='first'
|
|
322
|
+
Determines which duplicates (if any) to mark:
|
|
323
|
+
- 'first': Mark duplicates as True except for the first occurrence.
|
|
324
|
+
- 'last': Mark duplicates as True except for the last occurrence.
|
|
325
|
+
- False: Mark all duplicates as True.
|
|
326
|
+
|
|
327
|
+
return_records : bool, default=False
|
|
328
|
+
If True, returns a DataFrame containing the duplicate records.
|
|
329
|
+
If False (default), returns the standard duplicate summary report.
|
|
330
|
+
|
|
331
|
+
Returns
|
|
332
|
+
-------
|
|
333
|
+
pandas.DataFrame
|
|
334
|
+
Report containing:
|
|
335
|
+
- Total Rows
|
|
336
|
+
- Duplicate Rows
|
|
337
|
+
- Duplicate Percentage(%)
|
|
338
|
+
- Unique Rows
|
|
339
|
+
- Status
|
|
340
|
+
- Suggestion
|
|
341
|
+
|
|
342
|
+
Notes
|
|
343
|
+
-----
|
|
344
|
+
A row is considered a duplicate if an identical row occurs earlier
|
|
345
|
+
in the DataFrame (following pandas `df.duplicated()` semantics).
|
|
346
|
+
|
|
347
|
+
The input DataFrame is not modified.
|
|
348
|
+
"""
|
|
349
|
+
_validate_dataframe(df)
|
|
350
|
+
|
|
351
|
+
if not isinstance(return_records, bool):
|
|
352
|
+
raise TypeError("return_records must be a boolean.")
|
|
353
|
+
|
|
354
|
+
if keep not in ("first", "last", False):
|
|
355
|
+
raise ValueError("keep must be one of 'first', 'last', or False.")
|
|
356
|
+
|
|
357
|
+
if subset is not None:
|
|
358
|
+
if isinstance(subset, str):
|
|
359
|
+
subset_cols = [subset]
|
|
360
|
+
elif hasattr(subset, "__iter__"):
|
|
361
|
+
subset_cols = list(subset)
|
|
362
|
+
else:
|
|
363
|
+
raise TypeError("subset must be a string, list of strings, or None.")
|
|
364
|
+
|
|
365
|
+
missing_cols = [col for col in subset_cols if col not in df.columns]
|
|
366
|
+
if missing_cols:
|
|
367
|
+
raise KeyError(f"Columns not found in DataFrame: {missing_cols}")
|
|
368
|
+
else:
|
|
369
|
+
subset_cols = None
|
|
370
|
+
|
|
371
|
+
total_rows = len(df)
|
|
372
|
+
if total_rows == 0:
|
|
373
|
+
duplicate_rows = 0
|
|
374
|
+
duplicate_percentage = 0.0
|
|
375
|
+
unique_rows = 0
|
|
376
|
+
status = "No Duplicates"
|
|
377
|
+
suggestion = "No action required"
|
|
378
|
+
dup_mask = pd.Series([], dtype=bool)
|
|
379
|
+
else:
|
|
380
|
+
try:
|
|
381
|
+
dup_mask = df.duplicated(subset=subset_cols, keep=keep)
|
|
382
|
+
duplicate_rows = int(dup_mask.sum())
|
|
383
|
+
except TypeError:
|
|
384
|
+
try:
|
|
385
|
+
# Map unhashable objects (lists, dicts, sets) to their string representations
|
|
386
|
+
map_fn = getattr(df, "map", getattr(df, "applymap", None))
|
|
387
|
+
if map_fn is not None:
|
|
388
|
+
safe_df = map_fn(lambda x: repr(x) if isinstance(x, (list, dict, set)) else x)
|
|
389
|
+
dup_mask = safe_df.duplicated(subset=subset_cols, keep=keep)
|
|
390
|
+
duplicate_rows = int(dup_mask.sum())
|
|
391
|
+
else:
|
|
392
|
+
dup_mask = pd.Series([False] * total_rows, index=df.index)
|
|
393
|
+
duplicate_rows = 0
|
|
394
|
+
except Exception:
|
|
395
|
+
dup_mask = pd.Series([False] * total_rows, index=df.index)
|
|
396
|
+
duplicate_rows = 0
|
|
397
|
+
|
|
398
|
+
duplicate_percentage = round((duplicate_rows / total_rows) * 100, 2)
|
|
399
|
+
unique_rows = total_rows - duplicate_rows
|
|
400
|
+
|
|
401
|
+
if duplicate_rows == 0:
|
|
402
|
+
status = "No Duplicates"
|
|
403
|
+
suggestion = "No action required"
|
|
404
|
+
else:
|
|
405
|
+
status = "Duplicates Found"
|
|
406
|
+
suggestion = "Consider reviewing and removing duplicate records"
|
|
407
|
+
|
|
408
|
+
if return_records:
|
|
409
|
+
if total_rows == 0:
|
|
410
|
+
return df.iloc[0:0].copy()
|
|
411
|
+
return df.iloc[dup_mask.to_numpy()].copy()
|
|
412
|
+
|
|
413
|
+
return pd.DataFrame(
|
|
414
|
+
[
|
|
415
|
+
{
|
|
416
|
+
"Total Rows": total_rows,
|
|
417
|
+
"Duplicate Rows": duplicate_rows,
|
|
418
|
+
"Duplicate Percentage(%)": duplicate_percentage,
|
|
419
|
+
"Unique Rows": unique_rows,
|
|
420
|
+
"Status": status,
|
|
421
|
+
"Suggestion": suggestion,
|
|
422
|
+
}
|
|
423
|
+
]
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def unique_report(df: pd.DataFrame, dropna: bool = False) -> pd.DataFrame:
|
|
428
|
+
"""
|
|
429
|
+
Generate a uniqueness and cardinality report for every column in a DataFrame.
|
|
430
|
+
|
|
431
|
+
Parameters
|
|
432
|
+
----------
|
|
433
|
+
df : pandas.DataFrame
|
|
434
|
+
Input DataFrame.
|
|
435
|
+
|
|
436
|
+
dropna : bool, default=False
|
|
437
|
+
Whether to exclude NaN/null values when counting unique values.
|
|
438
|
+
If False (default), NaN/null is counted as a distinct category if present.
|
|
439
|
+
|
|
440
|
+
Returns
|
|
441
|
+
-------
|
|
442
|
+
pandas.DataFrame
|
|
443
|
+
Report containing:
|
|
444
|
+
- Column
|
|
445
|
+
- Data Type
|
|
446
|
+
- Total Values
|
|
447
|
+
- Unique Values
|
|
448
|
+
- Unique Percentage(%)
|
|
449
|
+
- Duplicate Values
|
|
450
|
+
- Cardinality
|
|
451
|
+
- Status
|
|
452
|
+
- Suggestion
|
|
453
|
+
|
|
454
|
+
Notes
|
|
455
|
+
-----
|
|
456
|
+
Cardinality classifications:
|
|
457
|
+
- Constant: Exactly 1 unique value (Zero variance).
|
|
458
|
+
- Very High (Unique): 100% unique values (Potential unique identifier).
|
|
459
|
+
- Very High: >= 90% unique values (Near-unique / high diversity).
|
|
460
|
+
- High: 50% - 90% unique values.
|
|
461
|
+
- Medium: 10% - 50% unique values.
|
|
462
|
+
- Low: < 10% unique values.
|
|
463
|
+
|
|
464
|
+
The suggestions are heuristic recommendations and do not modify the input DataFrame.
|
|
465
|
+
"""
|
|
466
|
+
_validate_dataframe(df)
|
|
467
|
+
|
|
468
|
+
if not isinstance(dropna, bool):
|
|
469
|
+
raise TypeError("dropna must be a boolean.")
|
|
470
|
+
|
|
471
|
+
if df.shape[1] == 0:
|
|
472
|
+
return pd.DataFrame(
|
|
473
|
+
columns=[
|
|
474
|
+
"Column",
|
|
475
|
+
"Data Type",
|
|
476
|
+
"Total Values",
|
|
477
|
+
"Unique Values",
|
|
478
|
+
"Unique Percentage(%)",
|
|
479
|
+
"Duplicate Values",
|
|
480
|
+
"Cardinality",
|
|
481
|
+
"Status",
|
|
482
|
+
"Suggestion",
|
|
483
|
+
]
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
n_rows = len(df)
|
|
487
|
+
report_rows = []
|
|
488
|
+
|
|
489
|
+
for i in range(df.shape[1]):
|
|
490
|
+
column = df.columns[i]
|
|
491
|
+
series = df.iloc[:, i]
|
|
492
|
+
dtype_str = str(series.dtype)
|
|
493
|
+
|
|
494
|
+
if n_rows == 0:
|
|
495
|
+
report_rows.append(
|
|
496
|
+
{
|
|
497
|
+
"Column": column,
|
|
498
|
+
"Data Type": dtype_str,
|
|
499
|
+
"Total Values": 0,
|
|
500
|
+
"Unique Values": 0,
|
|
501
|
+
"Unique Percentage(%)": 0.0,
|
|
502
|
+
"Duplicate Values": 0,
|
|
503
|
+
"Cardinality": "Empty",
|
|
504
|
+
"Status": "Empty Column",
|
|
505
|
+
"Suggestion": "No action needed",
|
|
506
|
+
}
|
|
507
|
+
)
|
|
508
|
+
continue
|
|
509
|
+
|
|
510
|
+
try:
|
|
511
|
+
unique_count = int(series.nunique(dropna=dropna))
|
|
512
|
+
except TypeError:
|
|
513
|
+
try:
|
|
514
|
+
target = series if not dropna else series.dropna()
|
|
515
|
+
unique_count = int(len(set(map(repr, target))))
|
|
516
|
+
except Exception:
|
|
517
|
+
unique_count = n_rows
|
|
518
|
+
unique_pct = round((unique_count / n_rows) * 100, 2)
|
|
519
|
+
valid_count = int(series.count())
|
|
520
|
+
duplicate_count = (n_rows - unique_count) if not dropna else max(0, valid_count - unique_count)
|
|
521
|
+
|
|
522
|
+
col_str = str(column).lower().strip()
|
|
523
|
+
is_id_name = any(
|
|
524
|
+
x in col_str.replace("-", "_").split("_")
|
|
525
|
+
for x in ["id", "key", "code", "num", "number"]
|
|
526
|
+
) or col_str in ["id", "key", "code", "uuid", "guid"]
|
|
527
|
+
|
|
528
|
+
if unique_count == 0:
|
|
529
|
+
cardinality = "Empty"
|
|
530
|
+
status = "All Missing"
|
|
531
|
+
suggestion = "Column contains only missing values; consider dropping or imputing"
|
|
532
|
+
elif unique_count == 1:
|
|
533
|
+
cardinality = "Constant"
|
|
534
|
+
status = "Zero Variance"
|
|
535
|
+
suggestion = "Consider reviewing column; constant value across all rows"
|
|
536
|
+
elif unique_count == n_rows:
|
|
537
|
+
cardinality = "Very High (Unique)"
|
|
538
|
+
status = "Potential Identifier"
|
|
539
|
+
if pd.api.types.is_float_dtype(series):
|
|
540
|
+
suggestion = "Continuous numeric distribution with high uniqueness; review if feature or identifier"
|
|
541
|
+
else:
|
|
542
|
+
suggestion = "Review whether this column is a unique identifier"
|
|
543
|
+
elif unique_pct >= 90.0:
|
|
544
|
+
cardinality = "Very High"
|
|
545
|
+
status = "Near-Unique"
|
|
546
|
+
if is_id_name or (pd.api.types.is_integer_dtype(series) and not pd.api.types.is_bool_dtype(series)):
|
|
547
|
+
suggestion = "Review whether this column is an identifier, sequence, or high-cardinality feature"
|
|
548
|
+
elif pd.api.types.is_float_dtype(series):
|
|
549
|
+
suggestion = "Continuous numeric distribution with high uniqueness"
|
|
550
|
+
else:
|
|
551
|
+
suggestion = "Review whether this column is an identifier or high-cardinality feature"
|
|
552
|
+
elif unique_pct >= 50.0:
|
|
553
|
+
cardinality = "High"
|
|
554
|
+
status = "High Diversity"
|
|
555
|
+
if is_id_name:
|
|
556
|
+
suggestion = "High uniqueness discrete values; review whether this column is an identifier or key"
|
|
557
|
+
elif pd.api.types.is_float_dtype(series):
|
|
558
|
+
suggestion = "High uniqueness numeric distribution; review distribution spread"
|
|
559
|
+
elif pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(series):
|
|
560
|
+
suggestion = "High uniqueness discrete numeric values; review distribution"
|
|
561
|
+
else:
|
|
562
|
+
suggestion = "High cardinality categorical; review before one-hot encoding or grouping"
|
|
563
|
+
elif unique_pct >= 10.0:
|
|
564
|
+
cardinality = "Medium"
|
|
565
|
+
status = "Moderate Diversity"
|
|
566
|
+
suggestion = "Suitable for standard encoding or analysis"
|
|
567
|
+
else:
|
|
568
|
+
cardinality = "Low"
|
|
569
|
+
if unique_count == 2:
|
|
570
|
+
status = "Binary"
|
|
571
|
+
suggestion = "Suitable for binary encoding"
|
|
572
|
+
else:
|
|
573
|
+
status = "Low Diversity"
|
|
574
|
+
suggestion = "Suitable for categorical encoding"
|
|
575
|
+
|
|
576
|
+
report_rows.append(
|
|
577
|
+
{
|
|
578
|
+
"Column": column,
|
|
579
|
+
"Data Type": dtype_str,
|
|
580
|
+
"Total Values": n_rows,
|
|
581
|
+
"Unique Values": unique_count,
|
|
582
|
+
"Unique Percentage(%)": unique_pct,
|
|
583
|
+
"Duplicate Values": duplicate_count,
|
|
584
|
+
"Cardinality": cardinality,
|
|
585
|
+
"Status": status,
|
|
586
|
+
"Suggestion": suggestion,
|
|
587
|
+
}
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
return pd.DataFrame(report_rows)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def numeric_summary(
|
|
594
|
+
df: pd.DataFrame,
|
|
595
|
+
extended: bool = False,
|
|
596
|
+
percentiles: Optional[Sequence[float]] = None,
|
|
597
|
+
) -> pd.DataFrame:
|
|
598
|
+
"""
|
|
599
|
+
Generate a statistical summary for all numerical columns in a DataFrame.
|
|
600
|
+
|
|
601
|
+
Parameters
|
|
602
|
+
----------
|
|
603
|
+
df : pandas.DataFrame
|
|
604
|
+
Input DataFrame.
|
|
605
|
+
|
|
606
|
+
extended : bool, default=False
|
|
607
|
+
Whether to include extended statistical diagnostics:
|
|
608
|
+
- IQR (Interquartile Range)
|
|
609
|
+
- Skewness (Fisher-Pearson distribution asymmetry)
|
|
610
|
+
- Kurtosis (Fisher excess peakedness)
|
|
611
|
+
- Zeros (Count of zero values)
|
|
612
|
+
- Negatives (Count of negative values)
|
|
613
|
+
- Outliers (Count of observations beyond 1.5 * IQR)
|
|
614
|
+
|
|
615
|
+
percentiles : sequence of float, optional
|
|
616
|
+
Additional percentiles to calculate, e.g. [0.05, 0.95].
|
|
617
|
+
Values must be between 0.0 and 1.0.
|
|
618
|
+
|
|
619
|
+
Returns
|
|
620
|
+
-------
|
|
621
|
+
pandas.DataFrame
|
|
622
|
+
Summary report containing one row per numerical column:
|
|
623
|
+
- Column: Name of the numerical column
|
|
624
|
+
- Data Type: String representation of the column's data type
|
|
625
|
+
- Count: Number of non-null (valid) numerical observations
|
|
626
|
+
- Missing Values: Number of missing/null (NaN) entries
|
|
627
|
+
- Mean: Arithmetic average of valid observations
|
|
628
|
+
- Median: 50th percentile of valid observations
|
|
629
|
+
- Std: Sample standard deviation (ddof=1)
|
|
630
|
+
- Min: Minimum valid numerical value
|
|
631
|
+
- Q1: 25th percentile of valid observations
|
|
632
|
+
- Q3: 75th percentile of valid observations
|
|
633
|
+
- Max: Maximum valid numerical value
|
|
634
|
+
- Range: Spread between Maximum and Minimum (Max - Min)
|
|
635
|
+
[If extended=True: IQR, Skewness, Kurtosis, Zeros, Negatives, Outliers]
|
|
636
|
+
[If percentiles: Corresponding percentile columns, e.g. P5, P95]
|
|
637
|
+
|
|
638
|
+
Notes
|
|
639
|
+
-----
|
|
640
|
+
- Numerical Column Detection: Includes standard integer, float, nullable integer,
|
|
641
|
+
and nullable float dtypes. Boolean and complex columns are treated as non-numerical
|
|
642
|
+
and excluded from the summary.
|
|
643
|
+
- Missing Values: Statistics are calculated ignoring NaN values. Columns with 0 valid
|
|
644
|
+
observations return NaN for statistical measures without crashing.
|
|
645
|
+
- Standard Deviation: Uses sample standard deviation with N - 1 degrees of freedom (ddof=1).
|
|
646
|
+
Columns with single valid observations return NaN for Std.
|
|
647
|
+
- Infinity: Infinite values (np.inf, -np.inf) are retained according to standard
|
|
648
|
+
Pandas semantics and not converted to NaN.
|
|
649
|
+
- Outlier Detection: Identifies observations falling outside [Q1 - 1.5*IQR, Q3 + 1.5*IQR].
|
|
650
|
+
- The input DataFrame is not modified.
|
|
651
|
+
"""
|
|
652
|
+
_validate_dataframe(df)
|
|
653
|
+
|
|
654
|
+
if not isinstance(extended, bool):
|
|
655
|
+
raise TypeError("extended must be a boolean.")
|
|
656
|
+
|
|
657
|
+
validated_percentiles: List[float] = []
|
|
658
|
+
if percentiles is not None:
|
|
659
|
+
if not hasattr(percentiles, "__iter__") or isinstance(percentiles, (str, bytes)):
|
|
660
|
+
raise TypeError("percentiles must be a sequence of floats between 0 and 1.")
|
|
661
|
+
for p in percentiles:
|
|
662
|
+
if not isinstance(p, (int, float)) or isinstance(p, bool) or not (0.0 <= p <= 1.0):
|
|
663
|
+
raise ValueError(f"Percentile {p} must be a number between 0 and 1.")
|
|
664
|
+
validated_percentiles.append(float(p))
|
|
665
|
+
|
|
666
|
+
base_columns = [
|
|
667
|
+
"Column",
|
|
668
|
+
"Data Type",
|
|
669
|
+
"Count",
|
|
670
|
+
"Missing Values",
|
|
671
|
+
"Mean",
|
|
672
|
+
"Median",
|
|
673
|
+
"Std",
|
|
674
|
+
"Min",
|
|
675
|
+
"Q1",
|
|
676
|
+
"Q3",
|
|
677
|
+
"Max",
|
|
678
|
+
"Range",
|
|
679
|
+
]
|
|
680
|
+
|
|
681
|
+
output_columns = list(base_columns)
|
|
682
|
+
if extended:
|
|
683
|
+
output_columns.extend(["IQR", "Skewness", "Kurtosis", "Zeros", "Negatives", "Outliers"])
|
|
684
|
+
for p in validated_percentiles:
|
|
685
|
+
p_pct = p * 100
|
|
686
|
+
p_col = f"P{int(p_pct)}" if p_pct.is_integer() else f"P{p_pct:.1f}"
|
|
687
|
+
if p_col not in output_columns:
|
|
688
|
+
output_columns.append(p_col)
|
|
689
|
+
|
|
690
|
+
if df.shape[1] == 0:
|
|
691
|
+
return pd.DataFrame(columns=output_columns)
|
|
692
|
+
|
|
693
|
+
report_rows = []
|
|
694
|
+
|
|
695
|
+
for i in range(df.shape[1]):
|
|
696
|
+
column = df.columns[i]
|
|
697
|
+
series = df.iloc[:, i]
|
|
698
|
+
|
|
699
|
+
# Numerical detection: integer/float (including nullable), excluding bool and complex
|
|
700
|
+
if not pd.api.types.is_numeric_dtype(series) or pd.api.types.is_bool_dtype(series) or pd.api.types.is_complex_dtype(series):
|
|
701
|
+
continue
|
|
702
|
+
|
|
703
|
+
valid_count = int(series.count())
|
|
704
|
+
missing_count = int(series.isna().sum())
|
|
705
|
+
|
|
706
|
+
with warnings.catch_warnings():
|
|
707
|
+
warnings.simplefilter("ignore", category=RuntimeWarning)
|
|
708
|
+
|
|
709
|
+
if valid_count == 0:
|
|
710
|
+
mean_val = np.nan
|
|
711
|
+
median_val = np.nan
|
|
712
|
+
std_val = np.nan
|
|
713
|
+
min_val = np.nan
|
|
714
|
+
q1_val = np.nan
|
|
715
|
+
q3_val = np.nan
|
|
716
|
+
max_val = np.nan
|
|
717
|
+
range_val = np.nan
|
|
718
|
+
iqr_val = np.nan
|
|
719
|
+
skew_val = np.nan
|
|
720
|
+
kurt_val = np.nan
|
|
721
|
+
zeros_val = 0
|
|
722
|
+
negatives_val = 0
|
|
723
|
+
outliers_val = 0
|
|
724
|
+
else:
|
|
725
|
+
mean_val = series.mean()
|
|
726
|
+
median_val = series.median()
|
|
727
|
+
std_val = series.std() if valid_count > 1 else np.nan
|
|
728
|
+
min_val = series.min()
|
|
729
|
+
q1_val = series.quantile(0.25)
|
|
730
|
+
q3_val = series.quantile(0.75)
|
|
731
|
+
max_val = series.max()
|
|
732
|
+
try:
|
|
733
|
+
range_val = max_val - min_val
|
|
734
|
+
except Exception:
|
|
735
|
+
range_val = np.nan
|
|
736
|
+
|
|
737
|
+
if extended:
|
|
738
|
+
try:
|
|
739
|
+
iqr_val = q3_val - q1_val if (pd.notna(q3_val) and pd.notna(q1_val)) else np.nan
|
|
740
|
+
except Exception:
|
|
741
|
+
iqr_val = np.nan
|
|
742
|
+
|
|
743
|
+
skew_val = float(series.skew()) if valid_count >= 3 else np.nan
|
|
744
|
+
kurt_val = float(series.kurt()) if valid_count >= 4 else np.nan
|
|
745
|
+
|
|
746
|
+
try:
|
|
747
|
+
zeros_val = int((series == 0).sum())
|
|
748
|
+
except Exception:
|
|
749
|
+
zeros_val = 0
|
|
750
|
+
|
|
751
|
+
try:
|
|
752
|
+
negatives_val = int((series < 0).sum())
|
|
753
|
+
except Exception:
|
|
754
|
+
negatives_val = 0
|
|
755
|
+
|
|
756
|
+
if pd.notna(iqr_val) and not np.isinf(iqr_val) and iqr_val >= 0:
|
|
757
|
+
lower_bound = q1_val - 1.5 * iqr_val
|
|
758
|
+
upper_bound = q3_val + 1.5 * iqr_val
|
|
759
|
+
try:
|
|
760
|
+
outliers_val = int(((series < lower_bound) | (series > upper_bound)).sum())
|
|
761
|
+
except Exception:
|
|
762
|
+
outliers_val = 0
|
|
763
|
+
else:
|
|
764
|
+
outliers_val = 0
|
|
765
|
+
|
|
766
|
+
row_dict: dict[str, Any] = {
|
|
767
|
+
"Column": column,
|
|
768
|
+
"Data Type": str(series.dtype),
|
|
769
|
+
"Count": valid_count,
|
|
770
|
+
"Missing Values": missing_count,
|
|
771
|
+
"Mean": mean_val,
|
|
772
|
+
"Median": median_val,
|
|
773
|
+
"Std": std_val,
|
|
774
|
+
"Min": min_val,
|
|
775
|
+
"Q1": q1_val,
|
|
776
|
+
"Q3": q3_val,
|
|
777
|
+
"Max": max_val,
|
|
778
|
+
"Range": range_val,
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
if extended:
|
|
782
|
+
row_dict["IQR"] = iqr_val
|
|
783
|
+
row_dict["Skewness"] = skew_val
|
|
784
|
+
row_dict["Kurtosis"] = kurt_val
|
|
785
|
+
row_dict["Zeros"] = zeros_val
|
|
786
|
+
row_dict["Negatives"] = negatives_val
|
|
787
|
+
row_dict["Outliers"] = outliers_val
|
|
788
|
+
|
|
789
|
+
for p in validated_percentiles:
|
|
790
|
+
p_pct = p * 100
|
|
791
|
+
p_col = f"P{int(p_pct)}" if p_pct.is_integer() else f"P{p_pct:.1f}"
|
|
792
|
+
row_dict[p_col] = series.quantile(p) if valid_count > 0 else np.nan
|
|
793
|
+
|
|
794
|
+
report_rows.append(row_dict)
|
|
795
|
+
|
|
796
|
+
if not report_rows:
|
|
797
|
+
return pd.DataFrame(columns=output_columns)
|
|
798
|
+
|
|
799
|
+
return pd.DataFrame(report_rows)
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
# Convenience imports for cross-module accessibility
|
|
803
|
+
from .report import PrepixReport, report # noqa: E402, F401
|