cleancheck-cbdemo 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Your Name
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,97 @@
1
+ Metadata-Version: 2.4
2
+ Name: cleancheck_cbdemo
3
+ Version: 0.1.1
4
+ Summary: Simple data cleaning and data quality checks for pandas DataFrames
5
+ Author-email: Your Name <you@example.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/your-username/cleancheck
8
+ Keywords: data engineering,data cleaning,data quality,pandas,etl
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
12
+ Requires-Python: >=3.9
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE
15
+ Requires-Dist: pandas>=1.5
16
+ Provides-Extra: dev
17
+ Requires-Dist: pytest; extra == "dev"
18
+ Requires-Dist: build; extra == "dev"
19
+ Requires-Dist: twine; extra == "dev"
20
+ Dynamic: license-file
21
+
22
+ # cleancheck
23
+
24
+ Simple data cleaning and data quality checks for pandas DataFrames.
25
+ Built for everyday data engineering work: clean messy files, then check the data before loading it.
26
+
27
+ ## Install
28
+
29
+ ```bash
30
+ pip install cleancheck
31
+ ```
32
+
33
+ ## Quick start
34
+
35
+ ```python
36
+ import pandas as pd
37
+ import cleancheck as cc
38
+
39
+ df = pd.read_csv("orders.csv")
40
+
41
+ # Clean
42
+ df = cc.standardize_column_names(df)
43
+ df = cc.trim_whitespace(df)
44
+ df = cc.replace_blank_with_null(df)
45
+ df = cc.remove_duplicates(df, subset=["order_id"])
46
+ df = cc.convert_types(df, {"amount": "float", "order_date": "datetime"})
47
+
48
+ # Check
49
+ report = cc.run_checks([
50
+ cc.check_unique(df, "order_id"),
51
+ cc.check_not_null(df, "amount"),
52
+ cc.check_range(df, "amount", min_value=0),
53
+ cc.check_allowed_values(df, "status", ["shipped", "pending", "cancelled"]),
54
+ ])
55
+ print(report)
56
+
57
+ # Stop the pipeline if anything failed
58
+ cc.assert_quality([cc.check_unique(df, "order_id")])
59
+ ```
60
+
61
+ ## Cleaning functions
62
+
63
+ | Function | What it does |
64
+ |---|---|
65
+ | `standardize_column_names(df)` | `" Customer Name "` becomes `customer_name` |
66
+ | `trim_whitespace(df, columns=None)` | Removes extra spaces in text |
67
+ | `replace_blank_with_null(df)` | `""`, `"NA"`, `"null"`, `"-"` become real nulls |
68
+ | `remove_duplicates(df, subset=None)` | Drops repeated rows |
69
+ | `drop_empty(df)` | Drops fully empty rows and columns |
70
+ | `fill_missing(df, col, strategy, value)` | Fill nulls with value, mean, median, mode or ffill |
71
+ | `convert_types(df, type_map)` | Safe type change (int, float, datetime, str, bool). Bad values become null |
72
+ | `remove_outliers_iqr(df, col)` | Removes extreme numbers using the IQR rule |
73
+ | `standardize_text(df, col, case, mapping)` | Consistent case and fixes known spellings |
74
+
75
+ ## Quality checks
76
+
77
+ Each check returns a dict: `check`, `column`, `passed`, `failed_rows`, `details`.
78
+
79
+ | Function | What it checks |
80
+ |---|---|
81
+ | `check_not_null(df, col)` | No missing values |
82
+ | `check_unique(df, col)` | No repeated values (IDs) |
83
+ | `check_no_duplicate_rows(df, subset)` | No repeated rows |
84
+ | `check_range(df, col, min, max)` | Numbers inside limits |
85
+ | `check_allowed_values(df, col, allowed)` | Values from an allowed list |
86
+ | `check_pattern(df, col, regex)` | Text matches a regex |
87
+ | `check_email(df, col)` | Basic email format |
88
+ | `check_columns_exist(df, required)` | Required columns are present |
89
+ | `check_row_count(df, min_rows, max_rows)` | Table is not empty or too big |
90
+ | `check_freshness(df, col, max_age_days)` | Latest date is recent |
91
+ | `run_checks([...])` | Combine results into one table |
92
+ | `assert_quality([...])` | Raise an error if any check failed |
93
+ | `profile(df)` | Column overview: type, nulls, unique count, sample |
94
+
95
+ ## License
96
+
97
+ MIT
@@ -0,0 +1,76 @@
1
+ # cleancheck
2
+
3
+ Simple data cleaning and data quality checks for pandas DataFrames.
4
+ Built for everyday data engineering work: clean messy files, then check the data before loading it.
5
+
6
+ ## Install
7
+
8
+ ```bash
9
+ pip install cleancheck
10
+ ```
11
+
12
+ ## Quick start
13
+
14
+ ```python
15
+ import pandas as pd
16
+ import cleancheck as cc
17
+
18
+ df = pd.read_csv("orders.csv")
19
+
20
+ # Clean
21
+ df = cc.standardize_column_names(df)
22
+ df = cc.trim_whitespace(df)
23
+ df = cc.replace_blank_with_null(df)
24
+ df = cc.remove_duplicates(df, subset=["order_id"])
25
+ df = cc.convert_types(df, {"amount": "float", "order_date": "datetime"})
26
+
27
+ # Check
28
+ report = cc.run_checks([
29
+ cc.check_unique(df, "order_id"),
30
+ cc.check_not_null(df, "amount"),
31
+ cc.check_range(df, "amount", min_value=0),
32
+ cc.check_allowed_values(df, "status", ["shipped", "pending", "cancelled"]),
33
+ ])
34
+ print(report)
35
+
36
+ # Stop the pipeline if anything failed
37
+ cc.assert_quality([cc.check_unique(df, "order_id")])
38
+ ```
39
+
40
+ ## Cleaning functions
41
+
42
+ | Function | What it does |
43
+ |---|---|
44
+ | `standardize_column_names(df)` | `" Customer Name "` becomes `customer_name` |
45
+ | `trim_whitespace(df, columns=None)` | Removes extra spaces in text |
46
+ | `replace_blank_with_null(df)` | `""`, `"NA"`, `"null"`, `"-"` become real nulls |
47
+ | `remove_duplicates(df, subset=None)` | Drops repeated rows |
48
+ | `drop_empty(df)` | Drops fully empty rows and columns |
49
+ | `fill_missing(df, col, strategy, value)` | Fill nulls with value, mean, median, mode or ffill |
50
+ | `convert_types(df, type_map)` | Safe type change (int, float, datetime, str, bool). Bad values become null |
51
+ | `remove_outliers_iqr(df, col)` | Removes extreme numbers using the IQR rule |
52
+ | `standardize_text(df, col, case, mapping)` | Consistent case and fixes known spellings |
53
+
54
+ ## Quality checks
55
+
56
+ Each check returns a dict: `check`, `column`, `passed`, `failed_rows`, `details`.
57
+
58
+ | Function | What it checks |
59
+ |---|---|
60
+ | `check_not_null(df, col)` | No missing values |
61
+ | `check_unique(df, col)` | No repeated values (IDs) |
62
+ | `check_no_duplicate_rows(df, subset)` | No repeated rows |
63
+ | `check_range(df, col, min, max)` | Numbers inside limits |
64
+ | `check_allowed_values(df, col, allowed)` | Values from an allowed list |
65
+ | `check_pattern(df, col, regex)` | Text matches a regex |
66
+ | `check_email(df, col)` | Basic email format |
67
+ | `check_columns_exist(df, required)` | Required columns are present |
68
+ | `check_row_count(df, min_rows, max_rows)` | Table is not empty or too big |
69
+ | `check_freshness(df, col, max_age_days)` | Latest date is recent |
70
+ | `run_checks([...])` | Combine results into one table |
71
+ | `assert_quality([...])` | Raise an error if any check failed |
72
+ | `profile(df)` | Column overview: type, nulls, unique count, sample |
73
+
74
+ ## License
75
+
76
+ MIT
@@ -0,0 +1,30 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ # This is the name people use in "pip install". It must be unique on PyPI.
7
+ name = "cleancheck_cbdemo"
8
+ version = "0.1.1"
9
+ description = "Simple data cleaning and data quality checks for pandas DataFrames"
10
+ readme = "README.md"
11
+ requires-python = ">=3.9"
12
+ license = "MIT"
13
+ license-files = ["LICENSE"]
14
+ authors = [{ name = "Your Name", email = "you@example.com" }]
15
+ keywords = ["data engineering", "data cleaning", "data quality", "pandas", "etl"]
16
+ classifiers = [
17
+ "Programming Language :: Python :: 3",
18
+ "Operating System :: OS Independent",
19
+ "Topic :: Scientific/Engineering :: Information Analysis",
20
+ ]
21
+ dependencies = ["pandas>=1.5"]
22
+
23
+ [project.optional-dependencies]
24
+ dev = ["pytest", "build", "twine"]
25
+
26
+ [project.urls]
27
+ Homepage = "https://github.com/your-username/cleancheck"
28
+
29
+ [tool.setuptools.packages.find]
30
+ where = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,41 @@
1
+ """cleancheck: simple data cleaning and data quality checks for pandas."""
2
+
3
+ from .cleaning import (
4
+ convert_types,
5
+ drop_empty,
6
+ fill_missing,
7
+ remove_duplicates,
8
+ remove_outliers_iqr,
9
+ replace_blank_with_null,
10
+ standardize_column_names,
11
+ standardize_text,
12
+ trim_whitespace,
13
+ )
14
+ from .quality import (
15
+ assert_quality,
16
+ check_allowed_values,
17
+ check_columns_exist,
18
+ check_email,
19
+ check_freshness,
20
+ check_no_duplicate_rows,
21
+ check_not_null,
22
+ check_pattern,
23
+ check_range,
24
+ check_row_count,
25
+ check_unique,
26
+ profile,
27
+ run_checks,
28
+ )
29
+
30
+ __version__ = "0.1.0"
31
+
32
+ __all__ = [
33
+ # cleaning
34
+ "standardize_column_names", "trim_whitespace", "replace_blank_with_null",
35
+ "remove_duplicates", "drop_empty", "fill_missing", "convert_types",
36
+ "remove_outliers_iqr", "standardize_text",
37
+ # quality
38
+ "check_not_null", "check_unique", "check_no_duplicate_rows", "check_range",
39
+ "check_allowed_values", "check_pattern", "check_email", "check_columns_exist",
40
+ "check_row_count", "check_freshness", "run_checks", "assert_quality", "profile",
41
+ ]
@@ -0,0 +1,161 @@
1
+ """Common data cleaning functions for pandas DataFrames.
2
+
3
+ Every function returns a NEW DataFrame. Your original data is never changed.
4
+ """
5
+
6
+ import re
7
+
8
+ import pandas as pd
9
+
10
+
11
+ def standardize_column_names(df: pd.DataFrame) -> pd.DataFrame:
12
+ """Make column names lowercase snake_case.
13
+
14
+ Example: " Customer Name! " -> "customer_name"
15
+ """
16
+ out = df.copy()
17
+ new_cols = []
18
+ for col in out.columns:
19
+ name = str(col).strip().lower()
20
+ name = re.sub(r"[^0-9a-z]+", "_", name) # anything not a letter/number -> _
21
+ name = name.strip("_")
22
+ new_cols.append(name)
23
+ out.columns = new_cols
24
+ return out
25
+
26
+
27
+ def trim_whitespace(df: pd.DataFrame, columns: list | None = None) -> pd.DataFrame:
28
+ """Remove extra spaces from text columns.
29
+
30
+ Strips leading/trailing spaces and turns multiple spaces into one.
31
+ If columns is None, all text columns are cleaned.
32
+ """
33
+ out = df.copy()
34
+ cols = columns or out.select_dtypes(include=["object", "string"]).columns
35
+ for col in cols:
36
+ out[col] = out[col].map(
37
+ lambda v: re.sub(r"\s+", " ", v).strip() if isinstance(v, str) else v
38
+ )
39
+ return out
40
+
41
+
42
+ def replace_blank_with_null(df: pd.DataFrame) -> pd.DataFrame:
43
+ """Turn empty strings and placeholder text ("NA", "null", "-") into real nulls."""
44
+ placeholders = {"", "na", "n/a", "null", "none", "nan", "-", "?"}
45
+ out = df.copy()
46
+ for col in out.select_dtypes(include=["object", "string"]).columns:
47
+ out[col] = out[col].map(
48
+ lambda v: None if isinstance(v, str) and v.strip().lower() in placeholders else v
49
+ )
50
+ return out
51
+
52
+
53
+ def remove_duplicates(
54
+ df: pd.DataFrame, subset: list | None = None, keep: str = "first"
55
+ ) -> pd.DataFrame:
56
+ """Drop duplicate rows. Use subset to check only some columns (like an ID)."""
57
+ return df.drop_duplicates(subset=subset, keep=keep).reset_index(drop=True)
58
+
59
+
60
+ def drop_empty(df: pd.DataFrame) -> pd.DataFrame:
61
+ """Drop rows and columns where every value is null."""
62
+ return df.dropna(how="all").dropna(axis=1, how="all").reset_index(drop=True)
63
+
64
+
65
+ def fill_missing(
66
+ df: pd.DataFrame, column: str, strategy: str = "value", value=None
67
+ ) -> pd.DataFrame:
68
+ """Fill nulls in one column.
69
+
70
+ strategy:
71
+ "value" -> use the value you pass
72
+ "mean" -> column average (numbers only)
73
+ "median" -> column middle value (numbers only)
74
+ "mode" -> most common value
75
+ "ffill" -> copy the previous row's value
76
+ """
77
+ out = df.copy()
78
+ if strategy == "value":
79
+ fill = value
80
+ elif strategy == "mean":
81
+ fill = out[column].mean()
82
+ elif strategy == "median":
83
+ fill = out[column].median()
84
+ elif strategy == "mode":
85
+ modes = out[column].mode()
86
+ fill = modes.iloc[0] if not modes.empty else None
87
+ elif strategy == "ffill":
88
+ out[column] = out[column].ffill()
89
+ return out
90
+ else:
91
+ raise ValueError(
92
+ f"Unknown strategy '{strategy}'. Use value, mean, median, mode or ffill."
93
+ )
94
+ out[column] = out[column].fillna(fill)
95
+ return out
96
+
97
+
98
+ def convert_types(df: pd.DataFrame, type_map: dict) -> pd.DataFrame:
99
+ """Convert columns to a type. Bad values become null instead of crashing.
100
+
101
+ type_map example: {"age": "int", "price": "float", "order_date": "datetime",
102
+ "city": "str", "is_active": "bool"}
103
+ """
104
+ out = df.copy()
105
+ for col, dtype in type_map.items():
106
+ if dtype in ("int", "float"):
107
+ nums = pd.to_numeric(out[col], errors="coerce")
108
+ out[col] = nums.round().astype("Int64") if dtype == "int" else nums
109
+ elif dtype == "datetime":
110
+ out[col] = pd.to_datetime(out[col], errors="coerce")
111
+ elif dtype == "str":
112
+ out[col] = out[col].astype("string")
113
+ elif dtype == "bool":
114
+ true_vals = {"true", "yes", "y", "1"}
115
+ false_vals = {"false", "no", "n", "0"}
116
+ out[col] = out[col].map(
117
+ lambda v: True if str(v).strip().lower() in true_vals
118
+ else False if str(v).strip().lower() in false_vals
119
+ else None
120
+ ).astype("boolean")
121
+ else:
122
+ raise ValueError(f"Unknown type '{dtype}' for column '{col}'.")
123
+ return out
124
+
125
+
126
+ def remove_outliers_iqr(
127
+ df: pd.DataFrame, column: str, factor: float = 1.5
128
+ ) -> pd.DataFrame:
129
+ """Remove rows where a number is far from the normal range (IQR method).
130
+
131
+ Normal range = Q1 - factor*IQR to Q3 + factor*IQR. Nulls are kept.
132
+ """
133
+ q1 = df[column].quantile(0.25)
134
+ q3 = df[column].quantile(0.75)
135
+ iqr = q3 - q1
136
+ low, high = q1 - factor * iqr, q3 + factor * iqr
137
+ mask = df[column].between(low, high) | df[column].isna()
138
+ return df[mask].reset_index(drop=True)
139
+
140
+
141
+ def standardize_text(
142
+ df: pd.DataFrame, column: str, case: str = "lower", mapping: dict | None = None
143
+ ) -> pd.DataFrame:
144
+ """Make text values consistent.
145
+
146
+ case: "lower", "upper" or "title".
147
+ mapping: fix known variations, e.g. {"usa": "united states", "u.s.": "united states"}.
148
+ Mapping is applied after the case change, so write mapping keys in that case.
149
+ """
150
+ out = df.copy()
151
+ s = out[column].astype("string").str.strip()
152
+ if case == "lower":
153
+ s = s.str.lower()
154
+ elif case == "upper":
155
+ s = s.str.upper()
156
+ elif case == "title":
157
+ s = s.str.title()
158
+ if mapping:
159
+ s = s.replace(mapping)
160
+ out[column] = s
161
+ return out
@@ -0,0 +1,147 @@
1
+ """Common data quality checks for pandas DataFrames.
2
+
3
+ Every check returns a dict like:
4
+ {"check": "not_null", "column": "email", "passed": False,
5
+ "failed_rows": 3, "details": "3 null values"}
6
+
7
+ Use run_checks() to run many checks at once and get a summary table.
8
+ """
9
+
10
+ import pandas as pd
11
+
12
+ EMAIL_PATTERN = r"^[\w\.\+\-]+@[\w\-]+\.[\w\.\-]+$"
13
+
14
+
15
+ def _result(check: str, column, failed: int, details: str) -> dict:
16
+ return {
17
+ "check": check,
18
+ "column": column,
19
+ "passed": failed == 0,
20
+ "failed_rows": int(failed),
21
+ "details": details,
22
+ }
23
+
24
+
25
+ def check_not_null(df: pd.DataFrame, column: str) -> dict:
26
+ """Every value in the column must be present."""
27
+ failed = df[column].isna().sum()
28
+ return _result("not_null", column, failed, f"{failed} null values")
29
+
30
+
31
+ def check_unique(df: pd.DataFrame, column: str) -> dict:
32
+ """No value may appear more than once (good for IDs). Nulls are ignored."""
33
+ values = df[column].dropna()
34
+ failed = values.duplicated(keep=False).sum()
35
+ return _result("unique", column, failed, f"{failed} rows share a duplicate value")
36
+
37
+
38
+ def check_no_duplicate_rows(df: pd.DataFrame, subset: list | None = None) -> dict:
39
+ """No full row (or subset of columns) may be repeated."""
40
+ failed = df.duplicated(subset=subset).sum()
41
+ label = ",".join(subset) if subset else "ALL"
42
+ return _result("no_duplicate_rows", label, failed, f"{failed} duplicate rows")
43
+
44
+
45
+ def check_range(
46
+ df: pd.DataFrame, column: str, min_value=None, max_value=None
47
+ ) -> dict:
48
+ """Numbers must be between min_value and max_value (inclusive). Nulls are ignored."""
49
+ values = pd.to_numeric(df[column], errors="coerce").dropna()
50
+ bad = pd.Series(False, index=values.index)
51
+ if min_value is not None:
52
+ bad |= values < min_value
53
+ if max_value is not None:
54
+ bad |= values > max_value
55
+ failed = bad.sum()
56
+ return _result(
57
+ "range", column, failed, f"{failed} values outside [{min_value}, {max_value}]"
58
+ )
59
+
60
+
61
+ def check_allowed_values(df: pd.DataFrame, column: str, allowed: list) -> dict:
62
+ """Values must come from an allowed list. Nulls are ignored."""
63
+ values = df[column].dropna()
64
+ bad = values[~values.isin(allowed)]
65
+ examples = list(bad.unique()[:5])
66
+ return _result(
67
+ "allowed_values", column, len(bad), f"{len(bad)} invalid values, e.g. {examples}"
68
+ )
69
+
70
+
71
+ def check_pattern(df: pd.DataFrame, column: str, pattern: str) -> dict:
72
+ """Text values must match a regex pattern. Nulls are ignored."""
73
+ values = df[column].dropna().astype(str)
74
+ failed = (~values.str.match(pattern)).sum()
75
+ return _result("pattern", column, failed, f"{failed} values do not match pattern")
76
+
77
+
78
+ def check_email(df: pd.DataFrame, column: str) -> dict:
79
+ """Shortcut for a basic email format check."""
80
+ result = check_pattern(df, column, EMAIL_PATTERN)
81
+ result["check"] = "email_format"
82
+ return result
83
+
84
+
85
+ def check_columns_exist(df: pd.DataFrame, required: list) -> dict:
86
+ """All required columns must be present (a simple schema check)."""
87
+ missing = [c for c in required if c not in df.columns]
88
+ return _result("columns_exist", "SCHEMA", len(missing), f"missing columns: {missing}")
89
+
90
+
91
+ def check_row_count(df: pd.DataFrame, min_rows: int = 1, max_rows: int | None = None) -> dict:
92
+ """Row count must be within limits. Catches empty or exploded loads."""
93
+ n = len(df)
94
+ ok = n >= min_rows and (max_rows is None or n <= max_rows)
95
+ return _result(
96
+ "row_count", "TABLE", 0 if ok else 1,
97
+ f"{n} rows (expected min {min_rows}, max {max_rows})",
98
+ )
99
+
100
+
101
+ def check_freshness(df: pd.DataFrame, column: str, max_age_days: int) -> dict:
102
+ """The newest date in the column must be within max_age_days of today."""
103
+ dates = pd.to_datetime(df[column], errors="coerce")
104
+ latest = dates.max()
105
+ if pd.isna(latest):
106
+ return _result("freshness", column, 1, "no valid dates found")
107
+ age = (pd.Timestamp.now(tz=latest.tz) - latest).days
108
+ failed = 1 if age > max_age_days else 0
109
+ return _result("freshness", column, failed, f"latest date is {age} days old")
110
+
111
+
112
+ def run_checks(checks: list) -> pd.DataFrame:
113
+ """Combine check results into one summary table.
114
+
115
+ Example:
116
+ report = run_checks([
117
+ check_not_null(df, "id"),
118
+ check_unique(df, "id"),
119
+ check_range(df, "age", 0, 120),
120
+ ])
121
+ """
122
+ return pd.DataFrame(checks, columns=["check", "column", "passed", "failed_rows", "details"])
123
+
124
+
125
+ def assert_quality(checks: list) -> None:
126
+ """Raise an error if any check failed. Useful to stop a pipeline on bad data."""
127
+ report = run_checks(checks)
128
+ failed = report[~report["passed"]]
129
+ if not failed.empty:
130
+ raise ValueError("Data quality checks failed:\n" + failed.to_string(index=False))
131
+
132
+
133
+ def profile(df: pd.DataFrame) -> pd.DataFrame:
134
+ """Quick overview of every column: type, nulls, unique values, sample value."""
135
+ rows = []
136
+ for col in df.columns:
137
+ s = df[col]
138
+ non_null = s.dropna()
139
+ rows.append({
140
+ "column": col,
141
+ "dtype": str(s.dtype),
142
+ "null_count": int(s.isna().sum()),
143
+ "null_pct": round(s.isna().mean() * 100, 2),
144
+ "unique_count": int(s.nunique()),
145
+ "sample": non_null.iloc[0] if not non_null.empty else None,
146
+ })
147
+ return pd.DataFrame(rows)
@@ -0,0 +1,97 @@
1
+ Metadata-Version: 2.4
2
+ Name: cleancheck_cbdemo
3
+ Version: 0.1.1
4
+ Summary: Simple data cleaning and data quality checks for pandas DataFrames
5
+ Author-email: Your Name <you@example.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/your-username/cleancheck
8
+ Keywords: data engineering,data cleaning,data quality,pandas,etl
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
12
+ Requires-Python: >=3.9
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE
15
+ Requires-Dist: pandas>=1.5
16
+ Provides-Extra: dev
17
+ Requires-Dist: pytest; extra == "dev"
18
+ Requires-Dist: build; extra == "dev"
19
+ Requires-Dist: twine; extra == "dev"
20
+ Dynamic: license-file
21
+
22
+ # cleancheck
23
+
24
+ Simple data cleaning and data quality checks for pandas DataFrames.
25
+ Built for everyday data engineering work: clean messy files, then check the data before loading it.
26
+
27
+ ## Install
28
+
29
+ ```bash
30
+ pip install cleancheck
31
+ ```
32
+
33
+ ## Quick start
34
+
35
+ ```python
36
+ import pandas as pd
37
+ import cleancheck as cc
38
+
39
+ df = pd.read_csv("orders.csv")
40
+
41
+ # Clean
42
+ df = cc.standardize_column_names(df)
43
+ df = cc.trim_whitespace(df)
44
+ df = cc.replace_blank_with_null(df)
45
+ df = cc.remove_duplicates(df, subset=["order_id"])
46
+ df = cc.convert_types(df, {"amount": "float", "order_date": "datetime"})
47
+
48
+ # Check
49
+ report = cc.run_checks([
50
+ cc.check_unique(df, "order_id"),
51
+ cc.check_not_null(df, "amount"),
52
+ cc.check_range(df, "amount", min_value=0),
53
+ cc.check_allowed_values(df, "status", ["shipped", "pending", "cancelled"]),
54
+ ])
55
+ print(report)
56
+
57
+ # Stop the pipeline if anything failed
58
+ cc.assert_quality([cc.check_unique(df, "order_id")])
59
+ ```
60
+
61
+ ## Cleaning functions
62
+
63
+ | Function | What it does |
64
+ |---|---|
65
+ | `standardize_column_names(df)` | `" Customer Name "` becomes `customer_name` |
66
+ | `trim_whitespace(df, columns=None)` | Removes extra spaces in text |
67
+ | `replace_blank_with_null(df)` | `""`, `"NA"`, `"null"`, `"-"` become real nulls |
68
+ | `remove_duplicates(df, subset=None)` | Drops repeated rows |
69
+ | `drop_empty(df)` | Drops fully empty rows and columns |
70
+ | `fill_missing(df, col, strategy, value)` | Fill nulls with value, mean, median, mode or ffill |
71
+ | `convert_types(df, type_map)` | Safe type change (int, float, datetime, str, bool). Bad values become null |
72
+ | `remove_outliers_iqr(df, col)` | Removes extreme numbers using the IQR rule |
73
+ | `standardize_text(df, col, case, mapping)` | Consistent case and fixes known spellings |
74
+
75
+ ## Quality checks
76
+
77
+ Each check returns a dict: `check`, `column`, `passed`, `failed_rows`, `details`.
78
+
79
+ | Function | What it checks |
80
+ |---|---|
81
+ | `check_not_null(df, col)` | No missing values |
82
+ | `check_unique(df, col)` | No repeated values (IDs) |
83
+ | `check_no_duplicate_rows(df, subset)` | No repeated rows |
84
+ | `check_range(df, col, min, max)` | Numbers inside limits |
85
+ | `check_allowed_values(df, col, allowed)` | Values from an allowed list |
86
+ | `check_pattern(df, col, regex)` | Text matches a regex |
87
+ | `check_email(df, col)` | Basic email format |
88
+ | `check_columns_exist(df, required)` | Required columns are present |
89
+ | `check_row_count(df, min_rows, max_rows)` | Table is not empty or too big |
90
+ | `check_freshness(df, col, max_age_days)` | Latest date is recent |
91
+ | `run_checks([...])` | Combine results into one table |
92
+ | `assert_quality([...])` | Raise an error if any check failed |
93
+ | `profile(df)` | Column overview: type, nulls, unique count, sample |
94
+
95
+ ## License
96
+
97
+ MIT
@@ -0,0 +1,12 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ src/cleancheck/__init__.py
5
+ src/cleancheck/cleaning.py
6
+ src/cleancheck/quality.py
7
+ src/cleancheck_cbdemo.egg-info/PKG-INFO
8
+ src/cleancheck_cbdemo.egg-info/SOURCES.txt
9
+ src/cleancheck_cbdemo.egg-info/dependency_links.txt
10
+ src/cleancheck_cbdemo.egg-info/requires.txt
11
+ src/cleancheck_cbdemo.egg-info/top_level.txt
12
+ tests/test_cleancheck.py
@@ -0,0 +1,6 @@
1
+ pandas>=1.5
2
+
3
+ [dev]
4
+ pytest
5
+ build
6
+ twine
@@ -0,0 +1,78 @@
1
+ import pandas as pd
2
+ import pytest
3
+
4
+ import cleancheck as cc
5
+
6
+
7
+ @pytest.fixture
8
+ def raw():
9
+ return pd.DataFrame({
10
+ " Customer ID ": [1, 2, 2, 3, 4],
11
+ "Full Name": [" Asha Rao ", "Ben", "Ben", "N/A", "Dev"],
12
+ "Age": ["25", "abc", "abc", "40", "300"],
13
+ "Country": ["usa", "U.S.", "U.S.", "India", "india"],
14
+ "Email": ["asha@x.com", "bad-email", "bad-email", None, "dev@x.io"],
15
+ })
16
+
17
+
18
+ def test_standardize_column_names(raw):
19
+ out = cc.standardize_column_names(raw)
20
+ assert list(out.columns) == ["customer_id", "full_name", "age", "country", "email"]
21
+
22
+
23
+ def test_trim_and_blank(raw):
24
+ out = cc.replace_blank_with_null(cc.trim_whitespace(raw))
25
+ assert out["Full Name"][0] == "Asha Rao"
26
+ assert pd.isna(out["Full Name"][3])
27
+
28
+
29
+ def test_remove_duplicates(raw):
30
+ assert len(cc.remove_duplicates(raw)) == 4
31
+
32
+
33
+ def test_convert_types(raw):
34
+ out = cc.convert_types(raw, {"Age": "int"})
35
+ assert out["Age"].isna().sum() == 2
36
+ assert out["Age"][0] == 25
37
+
38
+
39
+ def test_fill_missing():
40
+ df = pd.DataFrame({"x": [1.0, None, 3.0]})
41
+ assert cc.fill_missing(df, "x", "mean")["x"][1] == 2.0
42
+ with pytest.raises(ValueError):
43
+ cc.fill_missing(df, "x", "wrong")
44
+
45
+
46
+ def test_remove_outliers():
47
+ df = pd.DataFrame({"v": [10, 11, 12, 13, 1000]})
48
+ assert 1000 not in cc.remove_outliers_iqr(df, "v")["v"].tolist()
49
+
50
+
51
+ def test_standardize_text(raw):
52
+ out = cc.standardize_text(raw, "Country", mapping={"u.s.": "usa"})
53
+ assert set(out["Country"]) == {"usa", "india"}
54
+
55
+
56
+ def test_quality_checks(raw):
57
+ df = cc.convert_types(raw, {"Age": "int"})
58
+ report = cc.run_checks([
59
+ cc.check_not_null(df, "Email"),
60
+ cc.check_unique(df, " Customer ID "),
61
+ cc.check_range(df, "Age", 0, 120),
62
+ cc.check_email(df, "Email"),
63
+ cc.check_allowed_values(df, "Country", ["usa", "india"]),
64
+ cc.check_columns_exist(df, ["Email", "Phone"]),
65
+ cc.check_row_count(df, min_rows=1),
66
+ ])
67
+ assert report["passed"].tolist() == [False, False, False, False, False, False, True]
68
+
69
+
70
+ def test_assert_quality_raises(raw):
71
+ with pytest.raises(ValueError):
72
+ cc.assert_quality([cc.check_not_null(raw, "Email")])
73
+
74
+
75
+ def test_profile(raw):
76
+ p = cc.profile(raw)
77
+ assert len(p) == 5
78
+ assert "null_pct" in p.columns