cleancheck-cbdemo 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cleancheck_cbdemo-0.1.1/LICENSE +21 -0
- cleancheck_cbdemo-0.1.1/PKG-INFO +97 -0
- cleancheck_cbdemo-0.1.1/README.md +76 -0
- cleancheck_cbdemo-0.1.1/pyproject.toml +30 -0
- cleancheck_cbdemo-0.1.1/setup.cfg +4 -0
- cleancheck_cbdemo-0.1.1/src/cleancheck/__init__.py +41 -0
- cleancheck_cbdemo-0.1.1/src/cleancheck/cleaning.py +161 -0
- cleancheck_cbdemo-0.1.1/src/cleancheck/quality.py +147 -0
- cleancheck_cbdemo-0.1.1/src/cleancheck_cbdemo.egg-info/PKG-INFO +97 -0
- cleancheck_cbdemo-0.1.1/src/cleancheck_cbdemo.egg-info/SOURCES.txt +12 -0
- cleancheck_cbdemo-0.1.1/src/cleancheck_cbdemo.egg-info/dependency_links.txt +1 -0
- cleancheck_cbdemo-0.1.1/src/cleancheck_cbdemo.egg-info/requires.txt +6 -0
- cleancheck_cbdemo-0.1.1/src/cleancheck_cbdemo.egg-info/top_level.txt +1 -0
- cleancheck_cbdemo-0.1.1/tests/test_cleancheck.py +78 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Your Name
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cleancheck_cbdemo
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Simple data cleaning and data quality checks for pandas DataFrames
|
|
5
|
+
Author-email: Your Name <you@example.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/your-username/cleancheck
|
|
8
|
+
Keywords: data engineering,data cleaning,data quality,pandas,etl
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
12
|
+
Requires-Python: >=3.9
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: pandas>=1.5
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest; extra == "dev"
|
|
18
|
+
Requires-Dist: build; extra == "dev"
|
|
19
|
+
Requires-Dist: twine; extra == "dev"
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# cleancheck
|
|
23
|
+
|
|
24
|
+
Simple data cleaning and data quality checks for pandas DataFrames.
|
|
25
|
+
Built for everyday data engineering work: clean messy files, then check the data before loading it.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install cleancheck
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Quick start
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
import pandas as pd
|
|
37
|
+
import cleancheck as cc
|
|
38
|
+
|
|
39
|
+
df = pd.read_csv("orders.csv")
|
|
40
|
+
|
|
41
|
+
# Clean
|
|
42
|
+
df = cc.standardize_column_names(df)
|
|
43
|
+
df = cc.trim_whitespace(df)
|
|
44
|
+
df = cc.replace_blank_with_null(df)
|
|
45
|
+
df = cc.remove_duplicates(df, subset=["order_id"])
|
|
46
|
+
df = cc.convert_types(df, {"amount": "float", "order_date": "datetime"})
|
|
47
|
+
|
|
48
|
+
# Check
|
|
49
|
+
report = cc.run_checks([
|
|
50
|
+
cc.check_unique(df, "order_id"),
|
|
51
|
+
cc.check_not_null(df, "amount"),
|
|
52
|
+
cc.check_range(df, "amount", min_value=0),
|
|
53
|
+
cc.check_allowed_values(df, "status", ["shipped", "pending", "cancelled"]),
|
|
54
|
+
])
|
|
55
|
+
print(report)
|
|
56
|
+
|
|
57
|
+
# Stop the pipeline if anything failed
|
|
58
|
+
cc.assert_quality([cc.check_unique(df, "order_id")])
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Cleaning functions
|
|
62
|
+
|
|
63
|
+
| Function | What it does |
|
|
64
|
+
|---|---|
|
|
65
|
+
| `standardize_column_names(df)` | `" Customer Name "` becomes `customer_name` |
|
|
66
|
+
| `trim_whitespace(df, columns=None)` | Removes extra spaces in text |
|
|
67
|
+
| `replace_blank_with_null(df)` | `""`, `"NA"`, `"null"`, `"-"` become real nulls |
|
|
68
|
+
| `remove_duplicates(df, subset=None)` | Drops repeated rows |
|
|
69
|
+
| `drop_empty(df)` | Drops fully empty rows and columns |
|
|
70
|
+
| `fill_missing(df, col, strategy, value)` | Fill nulls with value, mean, median, mode or ffill |
|
|
71
|
+
| `convert_types(df, type_map)` | Safe type change (int, float, datetime, str, bool). Bad values become null |
|
|
72
|
+
| `remove_outliers_iqr(df, col)` | Removes extreme numbers using the IQR rule |
|
|
73
|
+
| `standardize_text(df, col, case, mapping)` | Consistent case and fixes known spellings |
|
|
74
|
+
|
|
75
|
+
## Quality checks
|
|
76
|
+
|
|
77
|
+
Each check returns a dict: `check`, `column`, `passed`, `failed_rows`, `details`.
|
|
78
|
+
|
|
79
|
+
| Function | What it checks |
|
|
80
|
+
|---|---|
|
|
81
|
+
| `check_not_null(df, col)` | No missing values |
|
|
82
|
+
| `check_unique(df, col)` | No repeated values (IDs) |
|
|
83
|
+
| `check_no_duplicate_rows(df, subset)` | No repeated rows |
|
|
84
|
+
| `check_range(df, col, min, max)` | Numbers inside limits |
|
|
85
|
+
| `check_allowed_values(df, col, allowed)` | Values from an allowed list |
|
|
86
|
+
| `check_pattern(df, col, regex)` | Text matches a regex |
|
|
87
|
+
| `check_email(df, col)` | Basic email format |
|
|
88
|
+
| `check_columns_exist(df, required)` | Required columns are present |
|
|
89
|
+
| `check_row_count(df, min_rows, max_rows)` | Table is not empty or too big |
|
|
90
|
+
| `check_freshness(df, col, max_age_days)` | Latest date is recent |
|
|
91
|
+
| `run_checks([...])` | Combine results into one table |
|
|
92
|
+
| `assert_quality([...])` | Raise an error if any check failed |
|
|
93
|
+
| `profile(df)` | Column overview: type, nulls, unique count, sample |
|
|
94
|
+
|
|
95
|
+
## License
|
|
96
|
+
|
|
97
|
+
MIT
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# cleancheck
|
|
2
|
+
|
|
3
|
+
Simple data cleaning and data quality checks for pandas DataFrames.
|
|
4
|
+
Built for everyday data engineering work: clean messy files, then check the data before loading it.
|
|
5
|
+
|
|
6
|
+
## Install
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install cleancheck
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Quick start
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
import pandas as pd
|
|
16
|
+
import cleancheck as cc
|
|
17
|
+
|
|
18
|
+
df = pd.read_csv("orders.csv")
|
|
19
|
+
|
|
20
|
+
# Clean
|
|
21
|
+
df = cc.standardize_column_names(df)
|
|
22
|
+
df = cc.trim_whitespace(df)
|
|
23
|
+
df = cc.replace_blank_with_null(df)
|
|
24
|
+
df = cc.remove_duplicates(df, subset=["order_id"])
|
|
25
|
+
df = cc.convert_types(df, {"amount": "float", "order_date": "datetime"})
|
|
26
|
+
|
|
27
|
+
# Check
|
|
28
|
+
report = cc.run_checks([
|
|
29
|
+
cc.check_unique(df, "order_id"),
|
|
30
|
+
cc.check_not_null(df, "amount"),
|
|
31
|
+
cc.check_range(df, "amount", min_value=0),
|
|
32
|
+
cc.check_allowed_values(df, "status", ["shipped", "pending", "cancelled"]),
|
|
33
|
+
])
|
|
34
|
+
print(report)
|
|
35
|
+
|
|
36
|
+
# Stop the pipeline if anything failed
|
|
37
|
+
cc.assert_quality([cc.check_unique(df, "order_id")])
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Cleaning functions
|
|
41
|
+
|
|
42
|
+
| Function | What it does |
|
|
43
|
+
|---|---|
|
|
44
|
+
| `standardize_column_names(df)` | `" Customer Name "` becomes `customer_name` |
|
|
45
|
+
| `trim_whitespace(df, columns=None)` | Removes extra spaces in text |
|
|
46
|
+
| `replace_blank_with_null(df)` | `""`, `"NA"`, `"null"`, `"-"` become real nulls |
|
|
47
|
+
| `remove_duplicates(df, subset=None)` | Drops repeated rows |
|
|
48
|
+
| `drop_empty(df)` | Drops fully empty rows and columns |
|
|
49
|
+
| `fill_missing(df, col, strategy, value)` | Fill nulls with value, mean, median, mode or ffill |
|
|
50
|
+
| `convert_types(df, type_map)` | Safe type change (int, float, datetime, str, bool). Bad values become null |
|
|
51
|
+
| `remove_outliers_iqr(df, col)` | Removes extreme numbers using the IQR rule |
|
|
52
|
+
| `standardize_text(df, col, case, mapping)` | Consistent case and fixes known spellings |
|
|
53
|
+
|
|
54
|
+
## Quality checks
|
|
55
|
+
|
|
56
|
+
Each check returns a dict: `check`, `column`, `passed`, `failed_rows`, `details`.
|
|
57
|
+
|
|
58
|
+
| Function | What it checks |
|
|
59
|
+
|---|---|
|
|
60
|
+
| `check_not_null(df, col)` | No missing values |
|
|
61
|
+
| `check_unique(df, col)` | No repeated values (IDs) |
|
|
62
|
+
| `check_no_duplicate_rows(df, subset)` | No repeated rows |
|
|
63
|
+
| `check_range(df, col, min, max)` | Numbers inside limits |
|
|
64
|
+
| `check_allowed_values(df, col, allowed)` | Values from an allowed list |
|
|
65
|
+
| `check_pattern(df, col, regex)` | Text matches a regex |
|
|
66
|
+
| `check_email(df, col)` | Basic email format |
|
|
67
|
+
| `check_columns_exist(df, required)` | Required columns are present |
|
|
68
|
+
| `check_row_count(df, min_rows, max_rows)` | Table is not empty or too big |
|
|
69
|
+
| `check_freshness(df, col, max_age_days)` | Latest date is recent |
|
|
70
|
+
| `run_checks([...])` | Combine results into one table |
|
|
71
|
+
| `assert_quality([...])` | Raise an error if any check failed |
|
|
72
|
+
| `profile(df)` | Column overview: type, nulls, unique count, sample |
|
|
73
|
+
|
|
74
|
+
## License
|
|
75
|
+
|
|
76
|
+
MIT
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
# This is the name people use in "pip install". It must be unique on PyPI.
|
|
7
|
+
name = "cleancheck_cbdemo"
|
|
8
|
+
version = "0.1.1"
|
|
9
|
+
description = "Simple data cleaning and data quality checks for pandas DataFrames"
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
license = "MIT"
|
|
13
|
+
license-files = ["LICENSE"]
|
|
14
|
+
authors = [{ name = "Your Name", email = "you@example.com" }]
|
|
15
|
+
keywords = ["data engineering", "data cleaning", "data quality", "pandas", "etl"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
20
|
+
]
|
|
21
|
+
dependencies = ["pandas>=1.5"]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
dev = ["pytest", "build", "twine"]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/your-username/cleancheck"
|
|
28
|
+
|
|
29
|
+
[tool.setuptools.packages.find]
|
|
30
|
+
where = ["src"]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""cleancheck: simple data cleaning and data quality checks for pandas."""
|
|
2
|
+
|
|
3
|
+
from .cleaning import (
|
|
4
|
+
convert_types,
|
|
5
|
+
drop_empty,
|
|
6
|
+
fill_missing,
|
|
7
|
+
remove_duplicates,
|
|
8
|
+
remove_outliers_iqr,
|
|
9
|
+
replace_blank_with_null,
|
|
10
|
+
standardize_column_names,
|
|
11
|
+
standardize_text,
|
|
12
|
+
trim_whitespace,
|
|
13
|
+
)
|
|
14
|
+
from .quality import (
|
|
15
|
+
assert_quality,
|
|
16
|
+
check_allowed_values,
|
|
17
|
+
check_columns_exist,
|
|
18
|
+
check_email,
|
|
19
|
+
check_freshness,
|
|
20
|
+
check_no_duplicate_rows,
|
|
21
|
+
check_not_null,
|
|
22
|
+
check_pattern,
|
|
23
|
+
check_range,
|
|
24
|
+
check_row_count,
|
|
25
|
+
check_unique,
|
|
26
|
+
profile,
|
|
27
|
+
run_checks,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
__version__ = "0.1.0"
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
# cleaning
|
|
34
|
+
"standardize_column_names", "trim_whitespace", "replace_blank_with_null",
|
|
35
|
+
"remove_duplicates", "drop_empty", "fill_missing", "convert_types",
|
|
36
|
+
"remove_outliers_iqr", "standardize_text",
|
|
37
|
+
# quality
|
|
38
|
+
"check_not_null", "check_unique", "check_no_duplicate_rows", "check_range",
|
|
39
|
+
"check_allowed_values", "check_pattern", "check_email", "check_columns_exist",
|
|
40
|
+
"check_row_count", "check_freshness", "run_checks", "assert_quality", "profile",
|
|
41
|
+
]
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Common data cleaning functions for pandas DataFrames.
|
|
2
|
+
|
|
3
|
+
Every function returns a NEW DataFrame. Your original data is never changed.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def standardize_column_names(df: pd.DataFrame) -> pd.DataFrame:
|
|
12
|
+
"""Make column names lowercase snake_case.
|
|
13
|
+
|
|
14
|
+
Example: " Customer Name! " -> "customer_name"
|
|
15
|
+
"""
|
|
16
|
+
out = df.copy()
|
|
17
|
+
new_cols = []
|
|
18
|
+
for col in out.columns:
|
|
19
|
+
name = str(col).strip().lower()
|
|
20
|
+
name = re.sub(r"[^0-9a-z]+", "_", name) # anything not a letter/number -> _
|
|
21
|
+
name = name.strip("_")
|
|
22
|
+
new_cols.append(name)
|
|
23
|
+
out.columns = new_cols
|
|
24
|
+
return out
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def trim_whitespace(df: pd.DataFrame, columns: list | None = None) -> pd.DataFrame:
|
|
28
|
+
"""Remove extra spaces from text columns.
|
|
29
|
+
|
|
30
|
+
Strips leading/trailing spaces and turns multiple spaces into one.
|
|
31
|
+
If columns is None, all text columns are cleaned.
|
|
32
|
+
"""
|
|
33
|
+
out = df.copy()
|
|
34
|
+
cols = columns or out.select_dtypes(include=["object", "string"]).columns
|
|
35
|
+
for col in cols:
|
|
36
|
+
out[col] = out[col].map(
|
|
37
|
+
lambda v: re.sub(r"\s+", " ", v).strip() if isinstance(v, str) else v
|
|
38
|
+
)
|
|
39
|
+
return out
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def replace_blank_with_null(df: pd.DataFrame) -> pd.DataFrame:
|
|
43
|
+
"""Turn empty strings and placeholder text ("NA", "null", "-") into real nulls."""
|
|
44
|
+
placeholders = {"", "na", "n/a", "null", "none", "nan", "-", "?"}
|
|
45
|
+
out = df.copy()
|
|
46
|
+
for col in out.select_dtypes(include=["object", "string"]).columns:
|
|
47
|
+
out[col] = out[col].map(
|
|
48
|
+
lambda v: None if isinstance(v, str) and v.strip().lower() in placeholders else v
|
|
49
|
+
)
|
|
50
|
+
return out
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def remove_duplicates(
|
|
54
|
+
df: pd.DataFrame, subset: list | None = None, keep: str = "first"
|
|
55
|
+
) -> pd.DataFrame:
|
|
56
|
+
"""Drop duplicate rows. Use subset to check only some columns (like an ID)."""
|
|
57
|
+
return df.drop_duplicates(subset=subset, keep=keep).reset_index(drop=True)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def drop_empty(df: pd.DataFrame) -> pd.DataFrame:
|
|
61
|
+
"""Drop rows and columns where every value is null."""
|
|
62
|
+
return df.dropna(how="all").dropna(axis=1, how="all").reset_index(drop=True)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def fill_missing(
|
|
66
|
+
df: pd.DataFrame, column: str, strategy: str = "value", value=None
|
|
67
|
+
) -> pd.DataFrame:
|
|
68
|
+
"""Fill nulls in one column.
|
|
69
|
+
|
|
70
|
+
strategy:
|
|
71
|
+
"value" -> use the value you pass
|
|
72
|
+
"mean" -> column average (numbers only)
|
|
73
|
+
"median" -> column middle value (numbers only)
|
|
74
|
+
"mode" -> most common value
|
|
75
|
+
"ffill" -> copy the previous row's value
|
|
76
|
+
"""
|
|
77
|
+
out = df.copy()
|
|
78
|
+
if strategy == "value":
|
|
79
|
+
fill = value
|
|
80
|
+
elif strategy == "mean":
|
|
81
|
+
fill = out[column].mean()
|
|
82
|
+
elif strategy == "median":
|
|
83
|
+
fill = out[column].median()
|
|
84
|
+
elif strategy == "mode":
|
|
85
|
+
modes = out[column].mode()
|
|
86
|
+
fill = modes.iloc[0] if not modes.empty else None
|
|
87
|
+
elif strategy == "ffill":
|
|
88
|
+
out[column] = out[column].ffill()
|
|
89
|
+
return out
|
|
90
|
+
else:
|
|
91
|
+
raise ValueError(
|
|
92
|
+
f"Unknown strategy '{strategy}'. Use value, mean, median, mode or ffill."
|
|
93
|
+
)
|
|
94
|
+
out[column] = out[column].fillna(fill)
|
|
95
|
+
return out
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def convert_types(df: pd.DataFrame, type_map: dict) -> pd.DataFrame:
|
|
99
|
+
"""Convert columns to a type. Bad values become null instead of crashing.
|
|
100
|
+
|
|
101
|
+
type_map example: {"age": "int", "price": "float", "order_date": "datetime",
|
|
102
|
+
"city": "str", "is_active": "bool"}
|
|
103
|
+
"""
|
|
104
|
+
out = df.copy()
|
|
105
|
+
for col, dtype in type_map.items():
|
|
106
|
+
if dtype in ("int", "float"):
|
|
107
|
+
nums = pd.to_numeric(out[col], errors="coerce")
|
|
108
|
+
out[col] = nums.round().astype("Int64") if dtype == "int" else nums
|
|
109
|
+
elif dtype == "datetime":
|
|
110
|
+
out[col] = pd.to_datetime(out[col], errors="coerce")
|
|
111
|
+
elif dtype == "str":
|
|
112
|
+
out[col] = out[col].astype("string")
|
|
113
|
+
elif dtype == "bool":
|
|
114
|
+
true_vals = {"true", "yes", "y", "1"}
|
|
115
|
+
false_vals = {"false", "no", "n", "0"}
|
|
116
|
+
out[col] = out[col].map(
|
|
117
|
+
lambda v: True if str(v).strip().lower() in true_vals
|
|
118
|
+
else False if str(v).strip().lower() in false_vals
|
|
119
|
+
else None
|
|
120
|
+
).astype("boolean")
|
|
121
|
+
else:
|
|
122
|
+
raise ValueError(f"Unknown type '{dtype}' for column '{col}'.")
|
|
123
|
+
return out
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def remove_outliers_iqr(
|
|
127
|
+
df: pd.DataFrame, column: str, factor: float = 1.5
|
|
128
|
+
) -> pd.DataFrame:
|
|
129
|
+
"""Remove rows where a number is far from the normal range (IQR method).
|
|
130
|
+
|
|
131
|
+
Normal range = Q1 - factor*IQR to Q3 + factor*IQR. Nulls are kept.
|
|
132
|
+
"""
|
|
133
|
+
q1 = df[column].quantile(0.25)
|
|
134
|
+
q3 = df[column].quantile(0.75)
|
|
135
|
+
iqr = q3 - q1
|
|
136
|
+
low, high = q1 - factor * iqr, q3 + factor * iqr
|
|
137
|
+
mask = df[column].between(low, high) | df[column].isna()
|
|
138
|
+
return df[mask].reset_index(drop=True)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def standardize_text(
|
|
142
|
+
df: pd.DataFrame, column: str, case: str = "lower", mapping: dict | None = None
|
|
143
|
+
) -> pd.DataFrame:
|
|
144
|
+
"""Make text values consistent.
|
|
145
|
+
|
|
146
|
+
case: "lower", "upper" or "title".
|
|
147
|
+
mapping: fix known variations, e.g. {"usa": "united states", "u.s.": "united states"}.
|
|
148
|
+
Mapping is applied after the case change, so write mapping keys in that case.
|
|
149
|
+
"""
|
|
150
|
+
out = df.copy()
|
|
151
|
+
s = out[column].astype("string").str.strip()
|
|
152
|
+
if case == "lower":
|
|
153
|
+
s = s.str.lower()
|
|
154
|
+
elif case == "upper":
|
|
155
|
+
s = s.str.upper()
|
|
156
|
+
elif case == "title":
|
|
157
|
+
s = s.str.title()
|
|
158
|
+
if mapping:
|
|
159
|
+
s = s.replace(mapping)
|
|
160
|
+
out[column] = s
|
|
161
|
+
return out
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Common data quality checks for pandas DataFrames.
|
|
2
|
+
|
|
3
|
+
Every check returns a dict like:
|
|
4
|
+
{"check": "not_null", "column": "email", "passed": False,
|
|
5
|
+
"failed_rows": 3, "details": "3 null values"}
|
|
6
|
+
|
|
7
|
+
Use run_checks() to run many checks at once and get a summary table.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
EMAIL_PATTERN = r"^[\w\.\+\-]+@[\w\-]+\.[\w\.\-]+$"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _result(check: str, column, failed: int, details: str) -> dict:
|
|
16
|
+
return {
|
|
17
|
+
"check": check,
|
|
18
|
+
"column": column,
|
|
19
|
+
"passed": failed == 0,
|
|
20
|
+
"failed_rows": int(failed),
|
|
21
|
+
"details": details,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def check_not_null(df: pd.DataFrame, column: str) -> dict:
|
|
26
|
+
"""Every value in the column must be present."""
|
|
27
|
+
failed = df[column].isna().sum()
|
|
28
|
+
return _result("not_null", column, failed, f"{failed} null values")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def check_unique(df: pd.DataFrame, column: str) -> dict:
|
|
32
|
+
"""No value may appear more than once (good for IDs). Nulls are ignored."""
|
|
33
|
+
values = df[column].dropna()
|
|
34
|
+
failed = values.duplicated(keep=False).sum()
|
|
35
|
+
return _result("unique", column, failed, f"{failed} rows share a duplicate value")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def check_no_duplicate_rows(df: pd.DataFrame, subset: list | None = None) -> dict:
|
|
39
|
+
"""No full row (or subset of columns) may be repeated."""
|
|
40
|
+
failed = df.duplicated(subset=subset).sum()
|
|
41
|
+
label = ",".join(subset) if subset else "ALL"
|
|
42
|
+
return _result("no_duplicate_rows", label, failed, f"{failed} duplicate rows")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def check_range(
|
|
46
|
+
df: pd.DataFrame, column: str, min_value=None, max_value=None
|
|
47
|
+
) -> dict:
|
|
48
|
+
"""Numbers must be between min_value and max_value (inclusive). Nulls are ignored."""
|
|
49
|
+
values = pd.to_numeric(df[column], errors="coerce").dropna()
|
|
50
|
+
bad = pd.Series(False, index=values.index)
|
|
51
|
+
if min_value is not None:
|
|
52
|
+
bad |= values < min_value
|
|
53
|
+
if max_value is not None:
|
|
54
|
+
bad |= values > max_value
|
|
55
|
+
failed = bad.sum()
|
|
56
|
+
return _result(
|
|
57
|
+
"range", column, failed, f"{failed} values outside [{min_value}, {max_value}]"
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def check_allowed_values(df: pd.DataFrame, column: str, allowed: list) -> dict:
|
|
62
|
+
"""Values must come from an allowed list. Nulls are ignored."""
|
|
63
|
+
values = df[column].dropna()
|
|
64
|
+
bad = values[~values.isin(allowed)]
|
|
65
|
+
examples = list(bad.unique()[:5])
|
|
66
|
+
return _result(
|
|
67
|
+
"allowed_values", column, len(bad), f"{len(bad)} invalid values, e.g. {examples}"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def check_pattern(df: pd.DataFrame, column: str, pattern: str) -> dict:
|
|
72
|
+
"""Text values must match a regex pattern. Nulls are ignored."""
|
|
73
|
+
values = df[column].dropna().astype(str)
|
|
74
|
+
failed = (~values.str.match(pattern)).sum()
|
|
75
|
+
return _result("pattern", column, failed, f"{failed} values do not match pattern")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def check_email(df: pd.DataFrame, column: str) -> dict:
|
|
79
|
+
"""Shortcut for a basic email format check."""
|
|
80
|
+
result = check_pattern(df, column, EMAIL_PATTERN)
|
|
81
|
+
result["check"] = "email_format"
|
|
82
|
+
return result
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def check_columns_exist(df: pd.DataFrame, required: list) -> dict:
|
|
86
|
+
"""All required columns must be present (a simple schema check)."""
|
|
87
|
+
missing = [c for c in required if c not in df.columns]
|
|
88
|
+
return _result("columns_exist", "SCHEMA", len(missing), f"missing columns: {missing}")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def check_row_count(df: pd.DataFrame, min_rows: int = 1, max_rows: int | None = None) -> dict:
|
|
92
|
+
"""Row count must be within limits. Catches empty or exploded loads."""
|
|
93
|
+
n = len(df)
|
|
94
|
+
ok = n >= min_rows and (max_rows is None or n <= max_rows)
|
|
95
|
+
return _result(
|
|
96
|
+
"row_count", "TABLE", 0 if ok else 1,
|
|
97
|
+
f"{n} rows (expected min {min_rows}, max {max_rows})",
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def check_freshness(df: pd.DataFrame, column: str, max_age_days: int) -> dict:
|
|
102
|
+
"""The newest date in the column must be within max_age_days of today."""
|
|
103
|
+
dates = pd.to_datetime(df[column], errors="coerce")
|
|
104
|
+
latest = dates.max()
|
|
105
|
+
if pd.isna(latest):
|
|
106
|
+
return _result("freshness", column, 1, "no valid dates found")
|
|
107
|
+
age = (pd.Timestamp.now(tz=latest.tz) - latest).days
|
|
108
|
+
failed = 1 if age > max_age_days else 0
|
|
109
|
+
return _result("freshness", column, failed, f"latest date is {age} days old")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def run_checks(checks: list) -> pd.DataFrame:
|
|
113
|
+
"""Combine check results into one summary table.
|
|
114
|
+
|
|
115
|
+
Example:
|
|
116
|
+
report = run_checks([
|
|
117
|
+
check_not_null(df, "id"),
|
|
118
|
+
check_unique(df, "id"),
|
|
119
|
+
check_range(df, "age", 0, 120),
|
|
120
|
+
])
|
|
121
|
+
"""
|
|
122
|
+
return pd.DataFrame(checks, columns=["check", "column", "passed", "failed_rows", "details"])
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def assert_quality(checks: list) -> None:
|
|
126
|
+
"""Raise an error if any check failed. Useful to stop a pipeline on bad data."""
|
|
127
|
+
report = run_checks(checks)
|
|
128
|
+
failed = report[~report["passed"]]
|
|
129
|
+
if not failed.empty:
|
|
130
|
+
raise ValueError("Data quality checks failed:\n" + failed.to_string(index=False))
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def profile(df: pd.DataFrame) -> pd.DataFrame:
|
|
134
|
+
"""Quick overview of every column: type, nulls, unique values, sample value."""
|
|
135
|
+
rows = []
|
|
136
|
+
for col in df.columns:
|
|
137
|
+
s = df[col]
|
|
138
|
+
non_null = s.dropna()
|
|
139
|
+
rows.append({
|
|
140
|
+
"column": col,
|
|
141
|
+
"dtype": str(s.dtype),
|
|
142
|
+
"null_count": int(s.isna().sum()),
|
|
143
|
+
"null_pct": round(s.isna().mean() * 100, 2),
|
|
144
|
+
"unique_count": int(s.nunique()),
|
|
145
|
+
"sample": non_null.iloc[0] if not non_null.empty else None,
|
|
146
|
+
})
|
|
147
|
+
return pd.DataFrame(rows)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cleancheck_cbdemo
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Simple data cleaning and data quality checks for pandas DataFrames
|
|
5
|
+
Author-email: Your Name <you@example.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/your-username/cleancheck
|
|
8
|
+
Keywords: data engineering,data cleaning,data quality,pandas,etl
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
12
|
+
Requires-Python: >=3.9
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: pandas>=1.5
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest; extra == "dev"
|
|
18
|
+
Requires-Dist: build; extra == "dev"
|
|
19
|
+
Requires-Dist: twine; extra == "dev"
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# cleancheck
|
|
23
|
+
|
|
24
|
+
Simple data cleaning and data quality checks for pandas DataFrames.
|
|
25
|
+
Built for everyday data engineering work: clean messy files, then check the data before loading it.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install cleancheck
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Quick start
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
import pandas as pd
|
|
37
|
+
import cleancheck as cc
|
|
38
|
+
|
|
39
|
+
df = pd.read_csv("orders.csv")
|
|
40
|
+
|
|
41
|
+
# Clean
|
|
42
|
+
df = cc.standardize_column_names(df)
|
|
43
|
+
df = cc.trim_whitespace(df)
|
|
44
|
+
df = cc.replace_blank_with_null(df)
|
|
45
|
+
df = cc.remove_duplicates(df, subset=["order_id"])
|
|
46
|
+
df = cc.convert_types(df, {"amount": "float", "order_date": "datetime"})
|
|
47
|
+
|
|
48
|
+
# Check
|
|
49
|
+
report = cc.run_checks([
|
|
50
|
+
cc.check_unique(df, "order_id"),
|
|
51
|
+
cc.check_not_null(df, "amount"),
|
|
52
|
+
cc.check_range(df, "amount", min_value=0),
|
|
53
|
+
cc.check_allowed_values(df, "status", ["shipped", "pending", "cancelled"]),
|
|
54
|
+
])
|
|
55
|
+
print(report)
|
|
56
|
+
|
|
57
|
+
# Stop the pipeline if anything failed
|
|
58
|
+
cc.assert_quality([cc.check_unique(df, "order_id")])
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Cleaning functions
|
|
62
|
+
|
|
63
|
+
| Function | What it does |
|
|
64
|
+
|---|---|
|
|
65
|
+
| `standardize_column_names(df)` | `" Customer Name "` becomes `customer_name` |
|
|
66
|
+
| `trim_whitespace(df, columns=None)` | Removes extra spaces in text |
|
|
67
|
+
| `replace_blank_with_null(df)` | `""`, `"NA"`, `"null"`, `"-"` become real nulls |
|
|
68
|
+
| `remove_duplicates(df, subset=None)` | Drops repeated rows |
|
|
69
|
+
| `drop_empty(df)` | Drops fully empty rows and columns |
|
|
70
|
+
| `fill_missing(df, col, strategy, value)` | Fill nulls with value, mean, median, mode or ffill |
|
|
71
|
+
| `convert_types(df, type_map)` | Safe type change (int, float, datetime, str, bool). Bad values become null |
|
|
72
|
+
| `remove_outliers_iqr(df, col)` | Removes extreme numbers using the IQR rule |
|
|
73
|
+
| `standardize_text(df, col, case, mapping)` | Consistent case and fixes known spellings |
|
|
74
|
+
|
|
75
|
+
## Quality checks
|
|
76
|
+
|
|
77
|
+
Each check returns a dict: `check`, `column`, `passed`, `failed_rows`, `details`.
|
|
78
|
+
|
|
79
|
+
| Function | What it checks |
|
|
80
|
+
|---|---|
|
|
81
|
+
| `check_not_null(df, col)` | No missing values |
|
|
82
|
+
| `check_unique(df, col)` | No repeated values (IDs) |
|
|
83
|
+
| `check_no_duplicate_rows(df, subset)` | No repeated rows |
|
|
84
|
+
| `check_range(df, col, min, max)` | Numbers inside limits |
|
|
85
|
+
| `check_allowed_values(df, col, allowed)` | Values from an allowed list |
|
|
86
|
+
| `check_pattern(df, col, regex)` | Text matches a regex |
|
|
87
|
+
| `check_email(df, col)` | Basic email format |
|
|
88
|
+
| `check_columns_exist(df, required)` | Required columns are present |
|
|
89
|
+
| `check_row_count(df, min_rows, max_rows)` | Table is not empty or too big |
|
|
90
|
+
| `check_freshness(df, col, max_age_days)` | Latest date is recent |
|
|
91
|
+
| `run_checks([...])` | Combine results into one table |
|
|
92
|
+
| `assert_quality([...])` | Raise an error if any check failed |
|
|
93
|
+
| `profile(df)` | Column overview: type, nulls, unique count, sample |
|
|
94
|
+
|
|
95
|
+
## License
|
|
96
|
+
|
|
97
|
+
MIT
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/cleancheck/__init__.py
|
|
5
|
+
src/cleancheck/cleaning.py
|
|
6
|
+
src/cleancheck/quality.py
|
|
7
|
+
src/cleancheck_cbdemo.egg-info/PKG-INFO
|
|
8
|
+
src/cleancheck_cbdemo.egg-info/SOURCES.txt
|
|
9
|
+
src/cleancheck_cbdemo.egg-info/dependency_links.txt
|
|
10
|
+
src/cleancheck_cbdemo.egg-info/requires.txt
|
|
11
|
+
src/cleancheck_cbdemo.egg-info/top_level.txt
|
|
12
|
+
tests/test_cleancheck.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cleancheck
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
import cleancheck as cc
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@pytest.fixture
|
|
8
|
+
def raw():
|
|
9
|
+
return pd.DataFrame({
|
|
10
|
+
" Customer ID ": [1, 2, 2, 3, 4],
|
|
11
|
+
"Full Name": [" Asha Rao ", "Ben", "Ben", "N/A", "Dev"],
|
|
12
|
+
"Age": ["25", "abc", "abc", "40", "300"],
|
|
13
|
+
"Country": ["usa", "U.S.", "U.S.", "India", "india"],
|
|
14
|
+
"Email": ["asha@x.com", "bad-email", "bad-email", None, "dev@x.io"],
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_standardize_column_names(raw):
|
|
19
|
+
out = cc.standardize_column_names(raw)
|
|
20
|
+
assert list(out.columns) == ["customer_id", "full_name", "age", "country", "email"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_trim_and_blank(raw):
|
|
24
|
+
out = cc.replace_blank_with_null(cc.trim_whitespace(raw))
|
|
25
|
+
assert out["Full Name"][0] == "Asha Rao"
|
|
26
|
+
assert pd.isna(out["Full Name"][3])
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_remove_duplicates(raw):
|
|
30
|
+
assert len(cc.remove_duplicates(raw)) == 4
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_convert_types(raw):
|
|
34
|
+
out = cc.convert_types(raw, {"Age": "int"})
|
|
35
|
+
assert out["Age"].isna().sum() == 2
|
|
36
|
+
assert out["Age"][0] == 25
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_fill_missing():
|
|
40
|
+
df = pd.DataFrame({"x": [1.0, None, 3.0]})
|
|
41
|
+
assert cc.fill_missing(df, "x", "mean")["x"][1] == 2.0
|
|
42
|
+
with pytest.raises(ValueError):
|
|
43
|
+
cc.fill_missing(df, "x", "wrong")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_remove_outliers():
|
|
47
|
+
df = pd.DataFrame({"v": [10, 11, 12, 13, 1000]})
|
|
48
|
+
assert 1000 not in cc.remove_outliers_iqr(df, "v")["v"].tolist()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_standardize_text(raw):
|
|
52
|
+
out = cc.standardize_text(raw, "Country", mapping={"u.s.": "usa"})
|
|
53
|
+
assert set(out["Country"]) == {"usa", "india"}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_quality_checks(raw):
|
|
57
|
+
df = cc.convert_types(raw, {"Age": "int"})
|
|
58
|
+
report = cc.run_checks([
|
|
59
|
+
cc.check_not_null(df, "Email"),
|
|
60
|
+
cc.check_unique(df, " Customer ID "),
|
|
61
|
+
cc.check_range(df, "Age", 0, 120),
|
|
62
|
+
cc.check_email(df, "Email"),
|
|
63
|
+
cc.check_allowed_values(df, "Country", ["usa", "india"]),
|
|
64
|
+
cc.check_columns_exist(df, ["Email", "Phone"]),
|
|
65
|
+
cc.check_row_count(df, min_rows=1),
|
|
66
|
+
])
|
|
67
|
+
assert report["passed"].tolist() == [False, False, False, False, False, False, True]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_assert_quality_raises(raw):
|
|
71
|
+
with pytest.raises(ValueError):
|
|
72
|
+
cc.assert_quality([cc.check_not_null(raw, "Email")])
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_profile(raw):
|
|
76
|
+
p = cc.profile(raw)
|
|
77
|
+
assert len(p) == 5
|
|
78
|
+
assert "null_pct" in p.columns
|