serverlex 1.0.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- serverlex-1.2.0/Clean.py +2 -0
- serverlex-1.2.0/DataCheck.py +2 -0
- serverlex-1.2.0/DriftDetector.py +2 -0
- serverlex-1.2.0/EnsembleBuilder.py +2 -0
- serverlex-1.2.0/Evaluation.py +2 -0
- serverlex-1.2.0/Explainability.py +2 -0
- serverlex-1.2.0/FeatureSelector.py +2 -0
- serverlex-1.2.0/ModelDeployer.py +2 -0
- serverlex-1.2.0/ModelHyperparameter.py +2 -0
- {serverlex-1.0.0/serverlex.egg-info → serverlex-1.2.0}/PKG-INFO +45 -27
- serverlex-1.2.0/Pipeline.py +2 -0
- {serverlex-1.0.0 → serverlex-1.2.0}/README.md +38 -19
- serverlex-1.2.0/Visualization.py +2 -0
- serverlex-1.2.0/pyproject.toml +39 -0
- serverlex-1.2.0/serverlex/__init__.py +27 -0
- serverlex-1.2.0/serverlex/diagnostics/__init__.py +6 -0
- serverlex-1.0.0/DataCheck.py → serverlex-1.2.0/serverlex/diagnostics/dataguard.py +61 -46
- serverlex-1.2.0/serverlex/diagnostics/drift.py +169 -0
- serverlex-1.0.0/Evaluation.py → serverlex-1.2.0/serverlex/diagnostics/evaluation.py +26 -6
- serverlex-1.2.0/serverlex/modeling/__init__.py +5 -0
- serverlex-1.2.0/serverlex/preprocessing/__init__.py +4 -0
- serverlex-1.0.0/Clean.py → serverlex-1.2.0/serverlex/preprocessing/clean.py +12 -6
- serverlex-1.0.0/Pipeline.py → serverlex-1.2.0/serverlex/preprocessing/pipeline.py +124 -50
- serverlex-1.2.0/serverlex/serving/__init__.py +3 -0
- serverlex-1.2.0/serverlex/serving/deployer.py +200 -0
- serverlex-1.2.0/serverlex/visualization/__init__.py +3 -0
- serverlex-1.0.0/Visualization.py → serverlex-1.2.0/serverlex/visualization/eda.py +1 -1
- {serverlex-1.0.0 → serverlex-1.2.0/serverlex.egg-info}/PKG-INFO +45 -27
- serverlex-1.2.0/serverlex.egg-info/SOURCES.txt +43 -0
- {serverlex-1.0.0 → serverlex-1.2.0}/serverlex.egg-info/requires.txt +5 -0
- {serverlex-1.0.0 → serverlex-1.2.0}/setup.py +3 -3
- serverlex-1.2.0/tests/test_clean.py +31 -0
- serverlex-1.2.0/tests/test_dataguard.py +60 -0
- serverlex-1.2.0/tests/test_deployer.py +99 -0
- serverlex-1.2.0/tests/test_drift.py +80 -0
- serverlex-1.2.0/tests/test_leakage.py +97 -0
- serverlex-1.2.0/tests/test_parity.py +101 -0
- serverlex-1.2.0/tests/test_pipeline.py +42 -0
- serverlex-1.0.0/DriftDetector.py +0 -90
- serverlex-1.0.0/ModelDeployer.py +0 -89
- serverlex-1.0.0/__init__.py +0 -27
- serverlex-1.0.0/serverlex/__init__.py +0 -34
- serverlex-1.0.0/serverlex.egg-info/SOURCES.txt +0 -21
- serverlex-1.0.0/test_real_datasets.py +0 -139
- /serverlex-1.0.0/Explainability.py → /serverlex-1.2.0/serverlex/diagnostics/explainability.py +0 -0
- /serverlex-1.0.0/EnsembleBuilder.py → /serverlex-1.2.0/serverlex/modeling/ensemble.py +0 -0
- /serverlex-1.0.0/ModelHyperparameter.py → /serverlex-1.2.0/serverlex/modeling/hyperparameter.py +0 -0
- /serverlex-1.0.0/FeatureSelector.py → /serverlex-1.2.0/serverlex/modeling/selector.py +0 -0
- {serverlex-1.0.0 → serverlex-1.2.0}/serverlex.egg-info/dependency_links.txt +0 -0
- {serverlex-1.0.0 → serverlex-1.2.0}/serverlex.egg-info/top_level.txt +0 -0
- {serverlex-1.0.0 → serverlex-1.2.0}/setup.cfg +0 -0
serverlex-1.2.0/Clean.py
ADDED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: serverlex
|
|
3
|
-
Version: 1.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 1.2.0
|
|
4
|
+
Summary: Production-Safe ML Preprocessing, Diagnostics, Tuning, and Serving Platform
|
|
5
5
|
Author: Serverlex Team
|
|
6
|
+
License: MIT
|
|
6
7
|
Classifier: Programming Language :: Python :: 3
|
|
7
8
|
Classifier: License :: OSI Approved :: MIT License
|
|
8
9
|
Classifier: Operating System :: OS Independent
|
|
@@ -15,23 +16,20 @@ Requires-Dist: scipy>=1.10.0
|
|
|
15
16
|
Requires-Dist: matplotlib>=3.7.0
|
|
16
17
|
Requires-Dist: scikit-learn>=1.3.0
|
|
17
18
|
Requires-Dist: joblib>=1.3.0
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
Dynamic: requires-dist
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
21
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
22
|
+
Requires-Dist: flake8>=6.0.0; extra == "dev"
|
|
23
23
|
Dynamic: requires-python
|
|
24
|
-
Dynamic: summary
|
|
25
24
|
|
|
26
25
|
# Serverlex ⚡
|
|
27
26
|
**The Production-Safe Machine Learning Preprocessing, Diagnostics, & Serving Platform**
|
|
28
27
|
|
|
29
28
|
[](https://www.python.org/)
|
|
30
|
-
[](https://pypi.org/project/serverlex/)
|
|
30
|
+
[](https://pypi.org/project/serverlex/)
|
|
31
31
|
[](https://opensource.org/licenses/MIT)
|
|
32
|
-
[](https://github.com/)
|
|
33
|
-
|
|
34
|
-
> 🚀 **Coming Soon to PyPI!** Serverlex is currently undergoing final release preparations and will soon be installable with a simple `pip install serverlex`. Star this repository to get notified when the initial PyPI package is published!
|
|
32
|
+
[](https://github.com/Uwais-ML/Serverlex)
|
|
35
33
|
|
|
36
34
|
**Serverlex** is an end-to-end Python library that takes raw, dirty data and transforms it into model-ready features, baseline benchmarks, tuned algorithms, and live production endpoints—all without data leakage or boilerplate code.
|
|
37
35
|
|
|
@@ -39,17 +37,18 @@ Dynamic: summary
|
|
|
39
37
|
|
|
40
38
|
## 📦 Installation
|
|
41
39
|
|
|
42
|
-
|
|
40
|
+
Install the official release directly from [PyPI](https://pypi.org/project/serverlex/):
|
|
41
|
+
|
|
43
42
|
```bash
|
|
44
43
|
pip install serverlex
|
|
45
44
|
```
|
|
46
45
|
|
|
47
|
-
###
|
|
46
|
+
### Or Install From Source / GitHub:
|
|
48
47
|
```bash
|
|
49
|
-
#
|
|
48
|
+
# Latest development branch from GitHub:
|
|
50
49
|
pip install git+https://github.com/Uwais-ML/Serverlex.git
|
|
51
50
|
|
|
52
|
-
# Or install
|
|
51
|
+
# Or clone and install in editable mode:
|
|
53
52
|
git clone https://github.com/Uwais-ML/Serverlex.git
|
|
54
53
|
cd Serverlex
|
|
55
54
|
pip install -e .
|
|
@@ -163,17 +162,31 @@ print("Predicted Churn:", loaded_deployer.predict(raw_input)[0])
|
|
|
163
162
|
|
|
164
163
|
| Module | Core Class | Primary Purpose |
|
|
165
164
|
|---|---|---|
|
|
166
|
-
| [`
|
|
167
|
-
| [`
|
|
168
|
-
| [`
|
|
169
|
-
| [`
|
|
170
|
-
| [`
|
|
171
|
-
| [`
|
|
172
|
-
| [`
|
|
173
|
-
| [`
|
|
174
|
-
| [`
|
|
175
|
-
| [`
|
|
176
|
-
| [`
|
|
165
|
+
| [`serverlex.preprocessing.clean`](serverlex/preprocessing/clean.py) | `Clean` | Headers, deduplication, IQR/Z-score outlier capping, format parsing. |
|
|
166
|
+
| [`serverlex.diagnostics.dataguard`](serverlex/diagnostics/dataguard.py) | `DataGuard` | Pre-training safety audit (ID detection, target leakage, class imbalance). |
|
|
167
|
+
| [`serverlex.visualization.eda`](serverlex/visualization/eda.py) | `Visualize` | EDA plots (missing, distribution, box, correlation) and real-time `plt.ion()`. |
|
|
168
|
+
| [`serverlex.preprocessing.pipeline`](serverlex/preprocessing/pipeline.py) | `MLPipeline` | Leakage-free `fit`/`transform`, datetime extraction, encoding, scaling. |
|
|
169
|
+
| [`serverlex.modeling.selector`](serverlex/modeling/selector.py) | `FeatureSelector` | Mutual Information ranking, collinearity pruning, variance threshold. |
|
|
170
|
+
| [`serverlex.modeling.hyperparameter`](serverlex/modeling/hyperparameter.py) | `ModelHyperparameter` | Non-nested `GridSearchCV` and unbiased Nested Cross-Validation. |
|
|
171
|
+
| [`serverlex.modeling.ensemble`](serverlex/modeling/ensemble.py) | `EnsembleBuilder` | Soft/Hard Voting and Stacking ensembles with meta-learners. |
|
|
172
|
+
| [`serverlex.diagnostics.evaluation`](serverlex/diagnostics/evaluation.py) | `ModelEvaluator` | Confusion matrix, ROC-AUC, PR curves, and regression residuals. |
|
|
173
|
+
| [`serverlex.diagnostics.explainability`](serverlex/diagnostics/explainability.py) | `ModelExplainer` | Native tree feature importances & permutation importance. |
|
|
174
|
+
| [`serverlex.serving.deployer`](serverlex/serving/deployer.py) | `ModelDeployer` | Pipeline-model artifact bundling, schema validation, and raw inference. |
|
|
175
|
+
| [`serverlex.diagnostics.drift`](serverlex/diagnostics/drift.py) | `DriftDetector` | KS-test, Chi-Square, PSI (Population Stability Index), and Wasserstein distance. |
|
|
176
|
+
|
|
177
|
+
---
|
|
178
|
+
|
|
179
|
+
## 🧪 Running Unit Tests & Benchmarks
|
|
180
|
+
|
|
181
|
+
Run the complete test suite locally:
|
|
182
|
+
```bash
|
|
183
|
+
pytest -v tests/
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Reproduce the exact Kaggle benchmarks:
|
|
187
|
+
```bash
|
|
188
|
+
python benchmarks/reproduce_benchmarks.py
|
|
189
|
+
```
|
|
177
190
|
|
|
178
191
|
---
|
|
179
192
|
|
|
@@ -183,6 +196,11 @@ Serverlex was verified on real-world Kaggle datasets without custom code changes
|
|
|
183
196
|
|
|
184
197
|
- **Customer Churn (`muhammadshahidazeem/customer-churn-dataset`)**:
|
|
185
198
|
- `DataGuard` detected `CustomerID` as 100% unique ID and recommended removal.
|
|
199
|
+
- Leakage-free preprocessing reached **99.42% accuracy** using Gradient Boosting.
|
|
200
|
+
|
|
201
|
+
- **Walmart Store Sales (`mikhail1681/walmart-sales`)**:
|
|
202
|
+
- Automatically extracted `year`, `month`, `day`, `dayofweek`, and `is_weekend` from dates.
|
|
203
|
+
- Achieved $R^2 = 0.9455$ and RMSE = 133,727 using Random Forest Regressor.
|
|
186
204
|
- Baseline Gradient Boosting achieved **99.42% accuracy**.
|
|
187
205
|
- `ModelExplainer` identified `support_calls` (0.277) and `total_spend` (0.235) as top drivers.
|
|
188
206
|
|
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
**The Production-Safe Machine Learning Preprocessing, Diagnostics, & Serving Platform**
|
|
3
3
|
|
|
4
4
|
[](https://www.python.org/)
|
|
5
|
-
[](https://pypi.org/project/serverlex/)
|
|
6
|
+
[](https://pypi.org/project/serverlex/)
|
|
6
7
|
[](https://opensource.org/licenses/MIT)
|
|
7
|
-
[](https://github.com/)
|
|
8
|
-
|
|
9
|
-
> 🚀 **Coming Soon to PyPI!** Serverlex is currently undergoing final release preparations and will soon be installable with a simple `pip install serverlex`. Star this repository to get notified when the initial PyPI package is published!
|
|
8
|
+
[](https://github.com/Uwais-ML/Serverlex)
|
|
10
9
|
|
|
11
10
|
**Serverlex** is an end-to-end Python library that takes raw, dirty data and transforms it into model-ready features, baseline benchmarks, tuned algorithms, and live production endpoints—all without data leakage or boilerplate code.
|
|
12
11
|
|
|
@@ -14,17 +13,18 @@
|
|
|
14
13
|
|
|
15
14
|
## 📦 Installation
|
|
16
15
|
|
|
17
|
-
|
|
16
|
+
Install the official release directly from [PyPI](https://pypi.org/project/serverlex/):
|
|
17
|
+
|
|
18
18
|
```bash
|
|
19
19
|
pip install serverlex
|
|
20
20
|
```
|
|
21
21
|
|
|
22
|
-
###
|
|
22
|
+
### Or Install From Source / GitHub:
|
|
23
23
|
```bash
|
|
24
|
-
#
|
|
24
|
+
# Latest development branch from GitHub:
|
|
25
25
|
pip install git+https://github.com/Uwais-ML/Serverlex.git
|
|
26
26
|
|
|
27
|
-
# Or install
|
|
27
|
+
# Or clone and install in editable mode:
|
|
28
28
|
git clone https://github.com/Uwais-ML/Serverlex.git
|
|
29
29
|
cd Serverlex
|
|
30
30
|
pip install -e .
|
|
@@ -138,17 +138,31 @@ print("Predicted Churn:", loaded_deployer.predict(raw_input)[0])
|
|
|
138
138
|
|
|
139
139
|
| Module | Core Class | Primary Purpose |
|
|
140
140
|
|---|---|---|
|
|
141
|
-
| [`
|
|
142
|
-
| [`
|
|
143
|
-
| [`
|
|
144
|
-
| [`
|
|
145
|
-
| [`
|
|
146
|
-
| [`
|
|
147
|
-
| [`
|
|
148
|
-
| [`
|
|
149
|
-
| [`
|
|
150
|
-
| [`
|
|
151
|
-
| [`
|
|
141
|
+
| [`serverlex.preprocessing.clean`](serverlex/preprocessing/clean.py) | `Clean` | Headers, deduplication, IQR/Z-score outlier capping, format parsing. |
|
|
142
|
+
| [`serverlex.diagnostics.dataguard`](serverlex/diagnostics/dataguard.py) | `DataGuard` | Pre-training safety audit (ID detection, target leakage, class imbalance). |
|
|
143
|
+
| [`serverlex.visualization.eda`](serverlex/visualization/eda.py) | `Visualize` | EDA plots (missing, distribution, box, correlation) and real-time `plt.ion()`. |
|
|
144
|
+
| [`serverlex.preprocessing.pipeline`](serverlex/preprocessing/pipeline.py) | `MLPipeline` | Leakage-free `fit`/`transform`, datetime extraction, encoding, scaling. |
|
|
145
|
+
| [`serverlex.modeling.selector`](serverlex/modeling/selector.py) | `FeatureSelector` | Mutual Information ranking, collinearity pruning, variance threshold. |
|
|
146
|
+
| [`serverlex.modeling.hyperparameter`](serverlex/modeling/hyperparameter.py) | `ModelHyperparameter` | Non-nested `GridSearchCV` and unbiased Nested Cross-Validation. |
|
|
147
|
+
| [`serverlex.modeling.ensemble`](serverlex/modeling/ensemble.py) | `EnsembleBuilder` | Soft/Hard Voting and Stacking ensembles with meta-learners. |
|
|
148
|
+
| [`serverlex.diagnostics.evaluation`](serverlex/diagnostics/evaluation.py) | `ModelEvaluator` | Confusion matrix, ROC-AUC, PR curves, and regression residuals. |
|
|
149
|
+
| [`serverlex.diagnostics.explainability`](serverlex/diagnostics/explainability.py) | `ModelExplainer` | Native tree feature importances & permutation importance. |
|
|
150
|
+
| [`serverlex.serving.deployer`](serverlex/serving/deployer.py) | `ModelDeployer` | Pipeline-model artifact bundling, schema validation, and raw inference. |
|
|
151
|
+
| [`serverlex.diagnostics.drift`](serverlex/diagnostics/drift.py) | `DriftDetector` | KS-test, Chi-Square, PSI (Population Stability Index), and Wasserstein distance. |
|
|
152
|
+
|
|
153
|
+
---
|
|
154
|
+
|
|
155
|
+
## 🧪 Running Unit Tests & Benchmarks
|
|
156
|
+
|
|
157
|
+
Run the complete test suite locally:
|
|
158
|
+
```bash
|
|
159
|
+
pytest -v tests/
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Reproduce the exact Kaggle benchmarks:
|
|
163
|
+
```bash
|
|
164
|
+
python benchmarks/reproduce_benchmarks.py
|
|
165
|
+
```
|
|
152
166
|
|
|
153
167
|
---
|
|
154
168
|
|
|
@@ -158,6 +172,11 @@ Serverlex was verified on real-world Kaggle datasets without custom code changes
|
|
|
158
172
|
|
|
159
173
|
- **Customer Churn (`muhammadshahidazeem/customer-churn-dataset`)**:
|
|
160
174
|
- `DataGuard` detected `CustomerID` as 100% unique ID and recommended removal.
|
|
175
|
+
- Leakage-free preprocessing reached **99.42% accuracy** using Gradient Boosting.
|
|
176
|
+
|
|
177
|
+
- **Walmart Store Sales (`mikhail1681/walmart-sales`)**:
|
|
178
|
+
- Automatically extracted `year`, `month`, `day`, `dayofweek`, and `is_weekend` from dates.
|
|
179
|
+
- Achieved $R^2 = 0.9455$ and RMSE = 133,727 using Random Forest Regressor.
|
|
161
180
|
- Baseline Gradient Boosting achieved **99.42% accuracy**.
|
|
162
181
|
- `ModelExplainer` identified `support_calls` (0.277) and `total_spend` (0.235) as top drivers.
|
|
163
182
|
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "serverlex"
|
|
7
|
+
version = "1.2.0"
|
|
8
|
+
description = "Production-Safe ML Preprocessing, Diagnostics, Tuning, and Serving Platform"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [{ name = "Serverlex Team" }]
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
requires-python = ">=3.9"
|
|
13
|
+
dependencies = [
|
|
14
|
+
"pandas>=2.0.0",
|
|
15
|
+
"numpy>=1.24.0",
|
|
16
|
+
"scipy>=1.10.0",
|
|
17
|
+
"matplotlib>=3.7.0",
|
|
18
|
+
"scikit-learn>=1.3.0",
|
|
19
|
+
"joblib>=1.3.0"
|
|
20
|
+
]
|
|
21
|
+
classifiers = [
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"License :: OSI Approved :: MIT License",
|
|
24
|
+
"Operating System :: OS Independent",
|
|
25
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence"
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
dev = [
|
|
30
|
+
"pytest>=7.0.0",
|
|
31
|
+
"pytest-cov>=4.0.0",
|
|
32
|
+
"flake8>=6.0.0"
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[tool.pytest.ini_options]
|
|
36
|
+
testpaths = ["tests"]
|
|
37
|
+
python_files = ["test_*.py"]
|
|
38
|
+
python_classes = ["Test*"]
|
|
39
|
+
python_functions = ["test_*"]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from serverlex.preprocessing.clean import Clean
|
|
2
|
+
from serverlex.preprocessing.pipeline import MLPipeline
|
|
3
|
+
from serverlex.diagnostics.dataguard import DataGuard
|
|
4
|
+
from serverlex.diagnostics.evaluation import ModelEvaluator
|
|
5
|
+
from serverlex.diagnostics.explainability import ModelExplainer
|
|
6
|
+
from serverlex.diagnostics.drift import DriftDetector
|
|
7
|
+
from serverlex.modeling.hyperparameter import ModelHyperparameter
|
|
8
|
+
from serverlex.modeling.ensemble import EnsembleBuilder
|
|
9
|
+
from serverlex.modeling.selector import FeatureSelector
|
|
10
|
+
from serverlex.visualization.eda import Visualize
|
|
11
|
+
from serverlex.serving.deployer import ModelDeployer
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"Clean",
|
|
15
|
+
"MLPipeline",
|
|
16
|
+
"DataGuard",
|
|
17
|
+
"ModelEvaluator",
|
|
18
|
+
"ModelExplainer",
|
|
19
|
+
"DriftDetector",
|
|
20
|
+
"ModelHyperparameter",
|
|
21
|
+
"EnsembleBuilder",
|
|
22
|
+
"FeatureSelector",
|
|
23
|
+
"Visualize",
|
|
24
|
+
"ModelDeployer",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
__version__ = "1.2.0"
|
|
@@ -12,8 +12,9 @@ if not logger.handlers:
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class DataGuard:
|
|
15
|
-
def __init__(self, df=None):
|
|
15
|
+
def __init__(self, df=None, target=None):
|
|
16
16
|
self.data=df.copy() if df is not None else None
|
|
17
|
+
self.target=target
|
|
17
18
|
self.warnings=[]
|
|
18
19
|
self.id_cols=[]
|
|
19
20
|
self.leakage_cols=[]
|
|
@@ -24,14 +25,14 @@ class DataGuard:
|
|
|
24
25
|
self.warnings=[]
|
|
25
26
|
|
|
26
27
|
# 1. Detect ID Columns
|
|
27
|
-
def check_id_columns(self):
|
|
28
|
+
def check_id_columns(self, threshold=0.95):
|
|
28
29
|
try:
|
|
29
30
|
self.id_cols=[]
|
|
30
31
|
for col in self.data.columns:
|
|
31
32
|
unique_ratio=self.data[col].nunique(dropna=True) / len(self.data)
|
|
32
33
|
is_name_id=bool(re.search(r'(id|uuid|key|ssn|index|identifier)$', col.lower()))
|
|
33
34
|
|
|
34
|
-
if (unique_ratio
|
|
35
|
+
if (unique_ratio >= threshold and is_name_id) or (unique_ratio == 1.0 and self.data[col].dtype == 'object'):
|
|
35
36
|
self.id_cols.append(col)
|
|
36
37
|
logger.warning("Potential unique ID column detected: '%s' (uniqueness: %.1f%%)", col, unique_ratio * 100)
|
|
37
38
|
|
|
@@ -45,114 +46,127 @@ class DataGuard:
|
|
|
45
46
|
return self.id_cols
|
|
46
47
|
except Exception as e:
|
|
47
48
|
logger.error("Error in check_id_columns: %s", e)
|
|
49
|
+
return []
|
|
48
50
|
|
|
49
51
|
# 2. Check Target Leakage
|
|
50
|
-
def check_target_leakage(self, target, threshold=0.
|
|
52
|
+
def check_target_leakage(self, target=None, threshold=0.90):
|
|
51
53
|
try:
|
|
52
54
|
self.leakage_cols=[]
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
55
|
+
target_to_use=target or self.target
|
|
56
|
+
if not target_to_use or target_to_use not in self.data.columns:
|
|
57
|
+
return []
|
|
58
|
+
|
|
59
|
+
num_cols=self.data.select_dtypes(include='number').copy()
|
|
60
|
+
if target_to_use not in num_cols.columns:
|
|
61
|
+
try:
|
|
62
|
+
num_cols[target_to_use]=pd.to_numeric(self.data[target_to_use])
|
|
63
|
+
except Exception:
|
|
64
|
+
num_cols[target_to_use]=self.data[target_to_use].astype('category').cat.codes
|
|
65
|
+
|
|
66
|
+
if target_to_use in num_cols.columns:
|
|
67
|
+
corrs=num_cols.corr().abs()[target_to_use]
|
|
59
68
|
for col, val in corrs.items():
|
|
60
|
-
if col !=
|
|
69
|
+
if col != target_to_use and val >= threshold:
|
|
61
70
|
self.leakage_cols.append((col, round(val, 4)))
|
|
62
|
-
logger.error("Suspicious target leakage: feature '%s' has correlation %.4f with target '%s'", col, val,
|
|
71
|
+
logger.error("Suspicious target leakage: feature '%s' has correlation %.4f with target '%s'", col, val, target_to_use)
|
|
63
72
|
|
|
64
73
|
if self.leakage_cols:
|
|
65
74
|
self.warnings.append({
|
|
66
75
|
"Check": "Target Leakage",
|
|
67
76
|
"Status": "FAIL",
|
|
68
|
-
"Details": f"Features {self.leakage_cols} have correlation >= {threshold} with target '{
|
|
77
|
+
"Details": f"Features {self.leakage_cols} have correlation >= {threshold} with target '{target_to_use}'.",
|
|
69
78
|
"Fix": "Verify if these columns contain future data."
|
|
70
79
|
})
|
|
71
80
|
return self.leakage_cols
|
|
72
81
|
except Exception as e:
|
|
73
82
|
logger.error("Error in check_target_leakage: %s", e)
|
|
83
|
+
return []
|
|
74
84
|
|
|
75
85
|
# 3. Check Class Imbalance
|
|
76
|
-
def check_class_imbalance(self, target):
|
|
86
|
+
def check_class_imbalance(self, target=None):
|
|
77
87
|
try:
|
|
78
|
-
|
|
88
|
+
target_to_use=target or self.target
|
|
89
|
+
if not target_to_use or target_to_use not in self.data.columns:
|
|
79
90
|
return
|
|
80
91
|
|
|
81
|
-
counts=self.data[
|
|
82
|
-
if len(counts) <= 20:
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
if max_prop >= 0.70:
|
|
88
|
-
status="WARN" if max_prop < 0.90 else "FAIL"
|
|
89
|
-
logger.warning("Class imbalance in '%s': dominant class is %.1f%% (ratio %.2f:1)", target, max_prop * 100, imbalance_ratio)
|
|
92
|
+
counts=self.data[target_to_use].value_counts(normalize=True)
|
|
93
|
+
if len(counts) > 1 and len(counts) <= 20:
|
|
94
|
+
minority_prop=counts.min()
|
|
95
|
+
if minority_prop < 0.10:
|
|
96
|
+
logger.warning("High class imbalance detected in '%s': minority class is %.2f%%", target_to_use, minority_prop * 100)
|
|
90
97
|
self.warnings.append({
|
|
91
98
|
"Check": "Class Imbalance",
|
|
92
|
-
"Status":
|
|
93
|
-
"Details": f"
|
|
94
|
-
"Fix": "Use
|
|
99
|
+
"Status": "WARN",
|
|
100
|
+
"Details": f"Minority class proportion is {minority_prop:.2%}.",
|
|
101
|
+
"Fix": "Use SMOTE, class weights, or stratified cross-validation."
|
|
95
102
|
})
|
|
96
103
|
except Exception as e:
|
|
97
104
|
logger.error("Error in check_class_imbalance: %s", e)
|
|
98
105
|
|
|
99
|
-
# 4. Check High
|
|
100
|
-
def check_cardinality(self, threshold=
|
|
106
|
+
# 4. Check High Cardinality
|
|
107
|
+
def check_cardinality(self, threshold=100):
|
|
101
108
|
try:
|
|
102
109
|
self.high_cardinality_cols=[]
|
|
103
|
-
cat_cols=self.data.select_dtypes(include=['object', 'category']).columns
|
|
110
|
+
cat_cols=self.data.select_dtypes(include=['object', 'category', 'string']).columns
|
|
104
111
|
|
|
105
112
|
for col in cat_cols:
|
|
106
113
|
n_unique=self.data[col].nunique()
|
|
107
|
-
if n_unique > threshold:
|
|
114
|
+
if n_unique > threshold and col not in self.id_cols:
|
|
108
115
|
self.high_cardinality_cols.append((col, n_unique))
|
|
109
|
-
logger.warning("High cardinality
|
|
116
|
+
logger.warning("High cardinality column '%s': %d unique values", col, n_unique)
|
|
110
117
|
|
|
111
118
|
if self.high_cardinality_cols:
|
|
112
119
|
self.warnings.append({
|
|
113
120
|
"Check": "High Cardinality",
|
|
114
121
|
"Status": "WARN",
|
|
115
|
-
"Details": f"
|
|
116
|
-
"Fix": "Use
|
|
122
|
+
"Details": f"Categorical columns {self.high_cardinality_cols} have > {threshold} unique categories.",
|
|
123
|
+
"Fix": "Use Target/Frequency Encoding or group rare categories."
|
|
117
124
|
})
|
|
125
|
+
return self.high_cardinality_cols
|
|
118
126
|
except Exception as e:
|
|
119
127
|
logger.error("Error in check_cardinality: %s", e)
|
|
128
|
+
return []
|
|
120
129
|
|
|
121
|
-
# 5. Check Infinite Values
|
|
130
|
+
# 5. Check Infinite or Extreme Values
|
|
122
131
|
def check_infinite_values(self):
|
|
123
132
|
try:
|
|
124
133
|
num_cols=self.data.select_dtypes(include='number').columns
|
|
125
134
|
inf_cols=[]
|
|
135
|
+
|
|
126
136
|
for col in num_cols:
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
137
|
+
inf_count=np.isinf(self.data[col]).sum()
|
|
138
|
+
if inf_count > 0:
|
|
139
|
+
inf_cols.append((col, int(inf_count)))
|
|
140
|
+
logger.error("Infinite values detected in '%s': %d occurrences", col, inf_count)
|
|
130
141
|
|
|
131
142
|
if inf_cols:
|
|
132
143
|
self.warnings.append({
|
|
133
144
|
"Check": "Infinite Values",
|
|
134
145
|
"Status": "FAIL",
|
|
135
|
-
"Details": f"Columns {inf_cols} contain inf
|
|
136
|
-
"Fix": "Replace
|
|
146
|
+
"Details": f"Columns {inf_cols} contain infinity (+inf or -inf).",
|
|
147
|
+
"Fix": "Replace infinite values with NaN or bounds."
|
|
137
148
|
})
|
|
149
|
+
return inf_cols
|
|
138
150
|
except Exception as e:
|
|
139
151
|
logger.error("Error in check_infinite_values: %s", e)
|
|
152
|
+
return []
|
|
140
153
|
|
|
141
154
|
# ==========================
|
|
142
155
|
# Master Health Audit Runner
|
|
143
156
|
# ==========================
|
|
144
|
-
def run_audit(self, target=None):
|
|
157
|
+
def run_audit(self, target=None, leakage_threshold=0.90, id_threshold=0.95, cardinality_threshold=100):
|
|
145
158
|
try:
|
|
146
159
|
self.warnings=[]
|
|
147
160
|
logger.info("Running data quality audit on %d rows, %d columns", len(self.data), len(self.data.columns))
|
|
148
161
|
|
|
149
|
-
self.check_id_columns()
|
|
162
|
+
self.check_id_columns(threshold=id_threshold)
|
|
150
163
|
self.check_infinite_values()
|
|
151
|
-
self.check_cardinality()
|
|
164
|
+
self.check_cardinality(threshold=cardinality_threshold)
|
|
152
165
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
self.
|
|
166
|
+
target_to_use=target or self.target
|
|
167
|
+
if target_to_use:
|
|
168
|
+
self.check_target_leakage(target=target_to_use, threshold=leakage_threshold)
|
|
169
|
+
self.check_class_imbalance(target=target_to_use)
|
|
156
170
|
|
|
157
171
|
if not self.warnings:
|
|
158
172
|
logger.info("Quality audit completed: all checks passed")
|
|
@@ -163,3 +177,4 @@ class DataGuard:
|
|
|
163
177
|
return report_df
|
|
164
178
|
except Exception as e:
|
|
165
179
|
logger.error("Error in run_audit: %s", e)
|
|
180
|
+
return pd.DataFrame()
|