serverlex 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- Clean.py +261 -0
- DataCheck.py +165 -0
- DriftDetector.py +90 -0
- EnsembleBuilder.py +78 -0
- Evaluation.py +191 -0
- Explainability.py +89 -0
- FeatureSelector.py +90 -0
- ModelDeployer.py +89 -0
- ModelHyperparameter.py +162 -0
- Pipeline.py +251 -0
- Visualization.py +362 -0
- serverlex/__init__.py +34 -0
- serverlex-1.0.0.dist-info/METADATA +208 -0
- serverlex-1.0.0.dist-info/RECORD +16 -0
- serverlex-1.0.0.dist-info/WHEEL +5 -0
- serverlex-1.0.0.dist-info/top_level.txt +12 -0
Pipeline.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import numpy as np
|
|
4
|
+
from Clean import Clean
|
|
5
|
+
|
|
6
|
+
from sklearn.model_selection import cross_val_score, KFold, StratifiedKFold
|
|
7
|
+
from sklearn.linear_model import LogisticRegression, Ridge
|
|
8
|
+
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
|
|
9
|
+
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor, GradientBoostingClassifier, GradientBoostingRegressor
|
|
10
|
+
|
|
11
|
+
logger=logging.getLogger("Pipeline")
|
|
12
|
+
if not logger.handlers:
|
|
13
|
+
_h=logging.StreamHandler()
|
|
14
|
+
_h.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", "%Y-%m-%d %H:%M:%S"))
|
|
15
|
+
logger.addHandler(_h)
|
|
16
|
+
logger.setLevel(logging.INFO)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class MLPipeline:
|
|
20
|
+
def __init__(self, data=None):
|
|
21
|
+
self.data=None
|
|
22
|
+
self.cleaner=None
|
|
23
|
+
self.target=None
|
|
24
|
+
self.feature_columns=[]
|
|
25
|
+
self.num_fill={}
|
|
26
|
+
self.cat_fill={}
|
|
27
|
+
self.scaler_mean={}
|
|
28
|
+
self.scaler_std={}
|
|
29
|
+
self.date_col=None
|
|
30
|
+
self.custom_models={}
|
|
31
|
+
|
|
32
|
+
if data is not None:
|
|
33
|
+
self.load_data(data)
|
|
34
|
+
|
|
35
|
+
def load_data(self, data):
|
|
36
|
+
if isinstance(data, str):
|
|
37
|
+
self.cleaner=Clean(data)
|
|
38
|
+
self.cleaner.load()
|
|
39
|
+
self.data=self.cleaner.data
|
|
40
|
+
elif isinstance(data, Clean):
|
|
41
|
+
self.cleaner=data
|
|
42
|
+
self.data=self.cleaner.data
|
|
43
|
+
elif isinstance(data, pd.DataFrame):
|
|
44
|
+
self.cleaner=Clean.__new__(Clean)
|
|
45
|
+
self.cleaner.data=data.copy()
|
|
46
|
+
self.cleaner.path=None
|
|
47
|
+
self.data=data
|
|
48
|
+
|
|
49
|
+
# ==========================
|
|
50
|
+
# 1. FIT (Learn & Save on self)
|
|
51
|
+
# ==========================
|
|
52
|
+
def fit(self, df, target=None, encode='onehot', scale='standard', data_time=None, numeric_string_cols=None, **kwargs):
|
|
53
|
+
try:
|
|
54
|
+
self.load_data(df)
|
|
55
|
+
self.target=target.strip().lower() if target else None
|
|
56
|
+
self.encode=encode
|
|
57
|
+
self.scale=scale
|
|
58
|
+
self.numeric_string_cols=[c.strip().lower().replace(' ', '_') for c in (numeric_string_cols or [])]
|
|
59
|
+
|
|
60
|
+
logger.info("Fitting pipeline on %d rows (target=%s, encode=%s, scale=%s)", len(self.data), self.target, self.encode, self.scale)
|
|
61
|
+
|
|
62
|
+
# 1. Clean the training data using your Clean class
|
|
63
|
+
self.cleaner.total_clean(data_time=data_time, numeric_string_cols=numeric_string_cols, **kwargs)
|
|
64
|
+
self.data=self.cleaner.data
|
|
65
|
+
self.cleaner.seperate()
|
|
66
|
+
|
|
67
|
+
# 2. Extract datetime features if provided
|
|
68
|
+
if data_time:
|
|
69
|
+
col=data_time[0].strip().lower().replace(' ', '_')
|
|
70
|
+
if col in self.data.columns:
|
|
71
|
+
self.date_col=col
|
|
72
|
+
dt=pd.to_datetime(self.data[col], format='mixed', errors='coerce')
|
|
73
|
+
self.data[f'{col}_year']=dt.dt.year
|
|
74
|
+
self.data[f'{col}_month']=dt.dt.month
|
|
75
|
+
self.data[f'{col}_day']=dt.dt.day
|
|
76
|
+
self.data[f'{col}_dayofweek']=dt.dt.dayofweek
|
|
77
|
+
self.data=self.data.drop(columns=[col])
|
|
78
|
+
|
|
79
|
+
self.cleaner.data=self.data
|
|
80
|
+
self.cleaner.seperate()
|
|
81
|
+
|
|
82
|
+
# 3. Memorize missing value fill values directly on self
|
|
83
|
+
for col in self.cleaner.numerical_columns.columns:
|
|
84
|
+
if col != self.target:
|
|
85
|
+
self.num_fill[col]=self.data[col].mean()
|
|
86
|
+
|
|
87
|
+
for col in self.cleaner.categrorical_columns.columns:
|
|
88
|
+
if col != self.target:
|
|
89
|
+
mode_val=self.data[col].mode()
|
|
90
|
+
self.cat_fill[col]=mode_val[0] if not mode_val.empty else 'unknown'
|
|
91
|
+
|
|
92
|
+
# 4. Encode categoricals to find final feature columns
|
|
93
|
+
train_X=self.data.copy()
|
|
94
|
+
if self.target and self.target in train_X.columns:
|
|
95
|
+
train_X=train_X.drop(columns=[self.target])
|
|
96
|
+
|
|
97
|
+
cat_cols=[c for c in self.cleaner.categrorical_columns.columns if c != self.target and c in train_X.columns]
|
|
98
|
+
if self.encode=='onehot' and cat_cols:
|
|
99
|
+
train_X=pd.get_dummies(train_X, columns=cat_cols, drop_first=True, dtype=int)
|
|
100
|
+
elif self.encode=='label' and cat_cols:
|
|
101
|
+
for col in cat_cols:
|
|
102
|
+
train_X[col]=train_X[col].astype('category').cat.codes
|
|
103
|
+
|
|
104
|
+
# Save exact list of final columns for future test data
|
|
105
|
+
self.feature_columns=train_X.columns.tolist()
|
|
106
|
+
|
|
107
|
+
# 5. Memorize scaling numbers on self
|
|
108
|
+
if self.scale=='standard':
|
|
109
|
+
for col in self.cleaner.numerical_columns.columns:
|
|
110
|
+
if col in self.feature_columns:
|
|
111
|
+
self.scaler_mean[col]=train_X[col].mean()
|
|
112
|
+
std_val=train_X[col].std()
|
|
113
|
+
self.scaler_std[col]=std_val if std_val != 0 else 1.0
|
|
114
|
+
|
|
115
|
+
logger.info("Fit complete. Feature count: %d", len(self.feature_columns))
|
|
116
|
+
return self
|
|
117
|
+
except Exception as e:
|
|
118
|
+
logger.error("Error during fit: %s", e)
|
|
119
|
+
return self
|
|
120
|
+
|
|
121
|
+
# ==========================
|
|
122
|
+
# 2. TRANSFORM (Uses self variables)
|
|
123
|
+
# ==========================
|
|
124
|
+
def transform(self, df):
|
|
125
|
+
try:
|
|
126
|
+
out=df.copy()
|
|
127
|
+
|
|
128
|
+
# 1. Clean column names
|
|
129
|
+
out.columns=out.columns.str.strip().str.lower().str.replace(' ', '_').str.replace(r'[^\w_]', '', regex=True)
|
|
130
|
+
|
|
131
|
+
# Clean numeric string columns using saved self.numeric_string_cols
|
|
132
|
+
if hasattr(self, 'numeric_string_cols'):
|
|
133
|
+
for col in self.numeric_string_cols:
|
|
134
|
+
if col in out.columns:
|
|
135
|
+
cleaned=out[col].astype(str).str.replace(r'[\$,]', '', regex=True)
|
|
136
|
+
out[col]=pd.to_numeric(cleaned, errors='coerce')
|
|
137
|
+
|
|
138
|
+
# 2. Extract datetime features using saved date_col
|
|
139
|
+
if self.date_col and self.date_col in out.columns:
|
|
140
|
+
dt=pd.to_datetime(out[self.date_col], format='mixed', errors='coerce')
|
|
141
|
+
out[f'{self.date_col}_year']=dt.dt.year
|
|
142
|
+
out[f'{self.date_col}_month']=dt.dt.month
|
|
143
|
+
out[f'{self.date_col}_day']=dt.dt.day
|
|
144
|
+
out[f'{self.date_col}_dayofweek']=dt.dt.dayofweek
|
|
145
|
+
out=out.drop(columns=[self.date_col])
|
|
146
|
+
|
|
147
|
+
# 3. Fill missing using memorized self.num_fill and self.cat_fill
|
|
148
|
+
for col, val in self.num_fill.items():
|
|
149
|
+
if col in out.columns:
|
|
150
|
+
out[col]=out[col].fillna(val)
|
|
151
|
+
|
|
152
|
+
for col, val in self.cat_fill.items():
|
|
153
|
+
if col in out.columns:
|
|
154
|
+
out[col]=out[col].astype(str).str.lower().str.replace(' ', '')
|
|
155
|
+
out[col]=out[col].fillna(val)
|
|
156
|
+
|
|
157
|
+
# 4. Check target
|
|
158
|
+
has_target=self.target and self.target in out.columns
|
|
159
|
+
target_series=out[self.target] if has_target else None
|
|
160
|
+
|
|
161
|
+
# 5. Encode categoricals
|
|
162
|
+
cat_cols=[c for c in self.cat_fill.keys() if c in out.columns]
|
|
163
|
+
if self.encode=='onehot' and cat_cols:
|
|
164
|
+
out=pd.get_dummies(out, columns=cat_cols, drop_first=True, dtype=int)
|
|
165
|
+
elif self.encode=='label' and cat_cols:
|
|
166
|
+
for col in cat_cols:
|
|
167
|
+
out[col]=out[col].astype('category').cat.codes
|
|
168
|
+
|
|
169
|
+
# 6. Reindex columns so test columns match self.feature_columns exactly
|
|
170
|
+
X=out.reindex(columns=self.feature_columns, fill_value=0)
|
|
171
|
+
|
|
172
|
+
# 7. Scale using memorized self.scaler_mean and self.scaler_std
|
|
173
|
+
if self.scale=='standard':
|
|
174
|
+
for col in self.scaler_mean.keys():
|
|
175
|
+
if col in X.columns:
|
|
176
|
+
X[col]=(X[col] - self.scaler_mean[col]) / self.scaler_std[col]
|
|
177
|
+
|
|
178
|
+
if has_target:
|
|
179
|
+
return X, target_series
|
|
180
|
+
return X
|
|
181
|
+
except Exception as e:
|
|
182
|
+
logger.error("Error during transform: %s", e)
|
|
183
|
+
return df
|
|
184
|
+
|
|
185
|
+
def fit_transform(self, df, **kwargs):
|
|
186
|
+
self.fit(df, **kwargs)
|
|
187
|
+
return self.transform(df)
|
|
188
|
+
|
|
189
|
+
# ==========================
|
|
190
|
+
# 3. Custom Models & Evaluation
|
|
191
|
+
# ==========================
|
|
192
|
+
def add_model(self, name, model):
|
|
193
|
+
self.custom_models[name]=model
|
|
194
|
+
logger.info("Registered custom model '%s'", name)
|
|
195
|
+
return self
|
|
196
|
+
|
|
197
|
+
def evaluate_baseline_models(self, X, y, cv=5):
|
|
198
|
+
try:
|
|
199
|
+
if not isinstance(y, pd.Series):
|
|
200
|
+
y=pd.Series(y)
|
|
201
|
+
|
|
202
|
+
is_classification=y.dtype=='object' or str(y.dtype).startswith('cat') or y.nunique() <= 20
|
|
203
|
+
task="Classification" if is_classification else "Regression"
|
|
204
|
+
|
|
205
|
+
results=[]
|
|
206
|
+
|
|
207
|
+
if is_classification:
|
|
208
|
+
kf=StratifiedKFold(n_splits=cv, shuffle=True, random_state=42)
|
|
209
|
+
models={
|
|
210
|
+
"Logistic Regression": LogisticRegression(max_iter=1000),
|
|
211
|
+
"Decision Tree": DecisionTreeClassifier(random_state=42),
|
|
212
|
+
"Random Forest": RandomForestClassifier(n_estimators=100, random_state=42),
|
|
213
|
+
"Gradient Boosting": GradientBoostingClassifier(random_state=42)
|
|
214
|
+
}
|
|
215
|
+
models.update(self.custom_models)
|
|
216
|
+
logger.info("Evaluating %d baseline models (task=%s, cv=%d)", len(models), task, cv)
|
|
217
|
+
|
|
218
|
+
for name, model in models.items():
|
|
219
|
+
try:
|
|
220
|
+
acc=cross_val_score(model, X, y, cv=kf, scoring="accuracy")
|
|
221
|
+
f1=cross_val_score(model, X, y, cv=kf, scoring="f1_weighted")
|
|
222
|
+
results.append({"Model": name, "Accuracy": round(acc.mean(), 4), "F1": round(f1.mean(), 4)})
|
|
223
|
+
except Exception as e:
|
|
224
|
+
results.append({"Model": name, "Accuracy": None, "F1": str(e)})
|
|
225
|
+
|
|
226
|
+
leaderboard=pd.DataFrame(results).sort_values(by="Accuracy", ascending=False)
|
|
227
|
+
else:
|
|
228
|
+
kf=KFold(n_splits=cv, shuffle=True, random_state=42)
|
|
229
|
+
models={
|
|
230
|
+
"Ridge Regression": Ridge(),
|
|
231
|
+
"Decision Tree": DecisionTreeRegressor(random_state=42),
|
|
232
|
+
"Random Forest": RandomForestRegressor(n_estimators=100, random_state=42),
|
|
233
|
+
"Gradient Boosting": GradientBoostingRegressor(random_state=42)
|
|
234
|
+
}
|
|
235
|
+
models.update(self.custom_models)
|
|
236
|
+
logger.info("Evaluating %d baseline models (task=%s, cv=%d)", len(models), task, cv)
|
|
237
|
+
|
|
238
|
+
for name, model in models.items():
|
|
239
|
+
try:
|
|
240
|
+
r2=cross_val_score(model, X, y, cv=kf, scoring="r2")
|
|
241
|
+
rmse=cross_val_score(model, X, y, cv=kf, scoring="neg_root_mean_squared_error")
|
|
242
|
+
results.append({"Model": name, "R2": round(r2.mean(), 4), "RMSE": round(-rmse.mean(), 4)})
|
|
243
|
+
except Exception as e:
|
|
244
|
+
results.append({"Model": name, "R2": None, "RMSE": str(e)})
|
|
245
|
+
|
|
246
|
+
leaderboard=pd.DataFrame(results).sort_values(by="R2", ascending=False)
|
|
247
|
+
|
|
248
|
+
logger.info("Baseline leaderboard:\n%s", leaderboard.to_string(index=False))
|
|
249
|
+
return leaderboard
|
|
250
|
+
except Exception as e:
|
|
251
|
+
logger.error("Error in evaluate_baseline_models: %s", e)
|
Visualization.py
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import matplotlib.pyplot as plt
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import numpy as np
|
|
5
|
+
from Clean import Clean
|
|
6
|
+
|
|
7
|
+
logger=logging.getLogger("Visualization")
|
|
8
|
+
if not logger.handlers:
|
|
9
|
+
_h=logging.StreamHandler()
|
|
10
|
+
_h.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", "%Y-%m-%d %H:%M:%S"))
|
|
11
|
+
logger.addHandler(_h)
|
|
12
|
+
logger.setLevel(logging.INFO)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Visualize:
|
|
16
|
+
def __init__(self, data=None):
|
|
17
|
+
"""
|
|
18
|
+
data can be:
|
|
19
|
+
- A CSV file path (str)
|
|
20
|
+
- An instance of Clean
|
|
21
|
+
- A pandas DataFrame
|
|
22
|
+
- None (data can be set or loaded later)
|
|
23
|
+
"""
|
|
24
|
+
self.data = None
|
|
25
|
+
self.cleaner = None
|
|
26
|
+
if data is not None:
|
|
27
|
+
self.set_data(data)
|
|
28
|
+
|
|
29
|
+
def set_data(self, data):
|
|
30
|
+
"""Sets or updates the active dataset."""
|
|
31
|
+
if isinstance(data, str):
|
|
32
|
+
self.cleaner = Clean(data)
|
|
33
|
+
self.cleaner.load()
|
|
34
|
+
self.data = self.cleaner.data
|
|
35
|
+
elif isinstance(data, Clean):
|
|
36
|
+
self.cleaner = data
|
|
37
|
+
if not hasattr(self.cleaner, "data") or self.cleaner.data is None:
|
|
38
|
+
self.cleaner.load()
|
|
39
|
+
self.data = self.cleaner.data
|
|
40
|
+
elif isinstance(data, pd.DataFrame):
|
|
41
|
+
self.cleaner = Clean.__new__(Clean)
|
|
42
|
+
self.cleaner.data = data.copy()
|
|
43
|
+
self.cleaner.path = None
|
|
44
|
+
self.data = data
|
|
45
|
+
|
|
46
|
+
def _sync(self):
|
|
47
|
+
"""Syncs self.data from self.cleaner if cleaner modified the dataframe."""
|
|
48
|
+
if self.cleaner is not None and hasattr(self.cleaner, "data") and self.cleaner.data is not None:
|
|
49
|
+
self.data = self.cleaner.data
|
|
50
|
+
|
|
51
|
+
# ==========================
|
|
52
|
+
# 1. Individual Plot Methods
|
|
53
|
+
# ==========================
|
|
54
|
+
|
|
55
|
+
def plot_missing(self, title="Missing Values per Column"):
|
|
56
|
+
"""Plots a bar chart showing the count of missing values for columns with NaNs."""
|
|
57
|
+
self._sync()
|
|
58
|
+
try:
|
|
59
|
+
if self.data is None:
|
|
60
|
+
print("No data available to plot.")
|
|
61
|
+
return
|
|
62
|
+
|
|
63
|
+
missing = self.data.isnull().sum()
|
|
64
|
+
missing = missing[missing > 0]
|
|
65
|
+
if missing.empty:
|
|
66
|
+
print("No missing values found in dataset.")
|
|
67
|
+
return
|
|
68
|
+
|
|
69
|
+
fig, ax = plt.subplots(figsize=(8, 4))
|
|
70
|
+
missing.plot(kind="bar", color="salmon", edgecolor="black", ax=ax)
|
|
71
|
+
ax.set_title(title, fontsize=12, fontweight="bold")
|
|
72
|
+
ax.set_ylabel("Count of NaNs")
|
|
73
|
+
ax.set_xlabel("Columns")
|
|
74
|
+
plt.xticks(rotation=45, ha="right")
|
|
75
|
+
plt.tight_layout()
|
|
76
|
+
plt.show()
|
|
77
|
+
except Exception as e:
|
|
78
|
+
print(e)
|
|
79
|
+
|
|
80
|
+
def plot_distribution(self, column=None, bins=20):
|
|
81
|
+
"""Plots histograms for one or all numerical columns."""
|
|
82
|
+
self._sync()
|
|
83
|
+
try:
|
|
84
|
+
if self.data is None:
|
|
85
|
+
print("No data available to plot.")
|
|
86
|
+
return
|
|
87
|
+
|
|
88
|
+
num_cols = self.data.select_dtypes(include="number").columns
|
|
89
|
+
if len(num_cols) == 0:
|
|
90
|
+
print("No numerical columns available.")
|
|
91
|
+
return
|
|
92
|
+
|
|
93
|
+
cols = [column] if column else num_cols
|
|
94
|
+
n_cols = len(cols)
|
|
95
|
+
fig, axes = plt.subplots(1, n_cols, figsize=(4 * n_cols, 4), squeeze=False)
|
|
96
|
+
for i, col in enumerate(cols):
|
|
97
|
+
if col in self.data.columns:
|
|
98
|
+
axes[0, i].hist(self.data[col].dropna(), bins=bins, color="skyblue", edgecolor="black")
|
|
99
|
+
axes[0, i].set_title(f"Dist: {col}")
|
|
100
|
+
axes[0, i].set_xlabel(col)
|
|
101
|
+
axes[0, i].set_ylabel("Frequency")
|
|
102
|
+
plt.tight_layout()
|
|
103
|
+
plt.show()
|
|
104
|
+
except Exception as e:
|
|
105
|
+
print(e)
|
|
106
|
+
|
|
107
|
+
def plot_box(self, columns=None, title="Outlier Detection (Box Plot)"):
|
|
108
|
+
"""Plots box plots for numeric columns to visualize outliers."""
|
|
109
|
+
self._sync()
|
|
110
|
+
try:
|
|
111
|
+
if self.data is None:
|
|
112
|
+
print("No data available to plot.")
|
|
113
|
+
return
|
|
114
|
+
|
|
115
|
+
num_df = self.data.select_dtypes(include="number")
|
|
116
|
+
if num_df.empty:
|
|
117
|
+
print("No numerical columns for box plot.")
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
if columns:
|
|
121
|
+
valid_cols = [c for c in columns if c in num_df.columns]
|
|
122
|
+
num_df = num_df[valid_cols]
|
|
123
|
+
|
|
124
|
+
fig, ax = plt.subplots(figsize=(max(6, len(num_df.columns) * 1.5), 5))
|
|
125
|
+
num_df.boxplot(ax=ax, patch_artist=True, boxprops=dict(facecolor="lightblue"))
|
|
126
|
+
ax.set_title(title, fontsize=12, fontweight="bold")
|
|
127
|
+
plt.xticks(rotation=45, ha="right")
|
|
128
|
+
plt.tight_layout()
|
|
129
|
+
plt.show()
|
|
130
|
+
except Exception as e:
|
|
131
|
+
print(e)
|
|
132
|
+
|
|
133
|
+
def plot_categorical(self, column=None, top_n=10, title=None):
|
|
134
|
+
"""Plots horizontal bar chart of top N categories."""
|
|
135
|
+
self._sync()
|
|
136
|
+
try:
|
|
137
|
+
if self.data is None:
|
|
138
|
+
print("No data available to plot.")
|
|
139
|
+
return
|
|
140
|
+
|
|
141
|
+
cat_cols = self.data.select_dtypes(include=["object", "category"]).columns
|
|
142
|
+
if len(cat_cols) == 0:
|
|
143
|
+
print("No categorical columns available.")
|
|
144
|
+
return
|
|
145
|
+
|
|
146
|
+
col = column if column else cat_cols[0]
|
|
147
|
+
if col not in self.data.columns:
|
|
148
|
+
print(f"Column '{col}' not found.")
|
|
149
|
+
return
|
|
150
|
+
|
|
151
|
+
counts = self.data[col].value_counts().head(top_n)
|
|
152
|
+
fig, ax = plt.subplots(figsize=(8, 4))
|
|
153
|
+
counts.plot(kind="barh", color="mediumaquamarine", edgecolor="black", ax=ax)
|
|
154
|
+
ax.invert_yaxis()
|
|
155
|
+
ax.set_title(title or f"Top {top_n} Categories in '{col}'", fontsize=12, fontweight="bold")
|
|
156
|
+
ax.set_xlabel("Count")
|
|
157
|
+
plt.tight_layout()
|
|
158
|
+
plt.show()
|
|
159
|
+
except Exception as e:
|
|
160
|
+
print(e)
|
|
161
|
+
|
|
162
|
+
def plot_correlation(self, title="Correlation Matrix Heatmap"):
|
|
163
|
+
"""Plots a correlation heatmap for numerical features using matplotlib."""
|
|
164
|
+
self._sync()
|
|
165
|
+
try:
|
|
166
|
+
if self.data is None:
|
|
167
|
+
print("No data available to plot.")
|
|
168
|
+
return
|
|
169
|
+
|
|
170
|
+
num_df = self.data.select_dtypes(include="number")
|
|
171
|
+
if num_df.shape[1] < 2:
|
|
172
|
+
print("Need at least 2 numerical columns for correlation plot.")
|
|
173
|
+
return
|
|
174
|
+
|
|
175
|
+
corr = num_df.corr()
|
|
176
|
+
fig, ax = plt.subplots(figsize=(max(6, len(corr) * 1.2), max(5, len(corr) * 1.0)))
|
|
177
|
+
cax = ax.imshow(corr, cmap="coolwarm", vmin=-1, vmax=1)
|
|
178
|
+
fig.colorbar(cax, ax=ax)
|
|
179
|
+
|
|
180
|
+
ax.set_xticks(range(len(corr.columns)))
|
|
181
|
+
ax.set_yticks(range(len(corr.columns)))
|
|
182
|
+
ax.set_xticklabels(corr.columns, rotation=45, ha="right")
|
|
183
|
+
ax.set_yticklabels(corr.columns)
|
|
184
|
+
|
|
185
|
+
# Annotate correlation values in cells
|
|
186
|
+
for i in range(len(corr)):
|
|
187
|
+
for j in range(len(corr.columns)):
|
|
188
|
+
val = corr.iloc[i, j]
|
|
189
|
+
ax.text(j, i, f"{val:.2f}", ha="center", va="center", color="black" if abs(val) < 0.7 else "white", fontsize=9)
|
|
190
|
+
|
|
191
|
+
ax.set_title(title, fontsize=12, fontweight="bold")
|
|
192
|
+
plt.tight_layout()
|
|
193
|
+
plt.show()
|
|
194
|
+
except Exception as e:
|
|
195
|
+
print(e)
|
|
196
|
+
|
|
197
|
+
def plot_scatter(self, x, y, title=None):
|
|
198
|
+
"""Plots a scatter plot between two numerical columns."""
|
|
199
|
+
self._sync()
|
|
200
|
+
try:
|
|
201
|
+
if self.data is None:
|
|
202
|
+
print("No data available to plot.")
|
|
203
|
+
return
|
|
204
|
+
|
|
205
|
+
if x not in self.data.columns or y not in self.data.columns:
|
|
206
|
+
print(f"One or both columns ({x}, {y}) not in DataFrame.")
|
|
207
|
+
return
|
|
208
|
+
|
|
209
|
+
fig, ax = plt.subplots(figsize=(6, 4))
|
|
210
|
+
ax.scatter(self.data[x], self.data[y], color="steelblue", alpha=0.7, edgecolors="k")
|
|
211
|
+
ax.set_title(title or f"{x} vs {y}", fontsize=12, fontweight="bold")
|
|
212
|
+
ax.set_xlabel(x)
|
|
213
|
+
ax.set_ylabel(y)
|
|
214
|
+
ax.grid(True, linestyle="--", alpha=0.5)
|
|
215
|
+
plt.tight_layout()
|
|
216
|
+
plt.show()
|
|
217
|
+
except Exception as e:
|
|
218
|
+
print(e)
|
|
219
|
+
|
|
220
|
+
# ==============================================================
|
|
221
|
+
# 2. Live Interactive Cleaning Pipeline with plt.ion()
|
|
222
|
+
# ==============================================================
|
|
223
|
+
|
|
224
|
+
def live_clean(
|
|
225
|
+
self,
|
|
226
|
+
cleaner=None,
|
|
227
|
+
data_time=None,
|
|
228
|
+
numeric_string_cols=None,
|
|
229
|
+
missing_param="m",
|
|
230
|
+
outlier_method="IQR",
|
|
231
|
+
outlier_param="m",
|
|
232
|
+
corr_target=None,
|
|
233
|
+
corr_threshold=0.1,
|
|
234
|
+
pause_time=1.5
|
|
235
|
+
):
|
|
236
|
+
"""
|
|
237
|
+
Runs the cleaning pipeline step-by-step with plt.ion() (interactive mode),
|
|
238
|
+
updating live plots at each stage to show before & after transformations.
|
|
239
|
+
"""
|
|
240
|
+
if cleaner is not None:
|
|
241
|
+
self.cleaner = cleaner
|
|
242
|
+
elif self.cleaner is None:
|
|
243
|
+
print("No cleaner instance provided.")
|
|
244
|
+
return
|
|
245
|
+
|
|
246
|
+
# Enable interactive plotting
|
|
247
|
+
plt.ion()
|
|
248
|
+
fig = plt.figure(figsize=(10, 6))
|
|
249
|
+
logger.info("Initializing live visual cleaning session")
|
|
250
|
+
|
|
251
|
+
try:
|
|
252
|
+
# 1. Load & Deduplicate
|
|
253
|
+
self.cleaner.load()
|
|
254
|
+
self.cleaner.clean_column_names()
|
|
255
|
+
self.cleaner.delete_duplicates()
|
|
256
|
+
self._sync()
|
|
257
|
+
|
|
258
|
+
# Step 1: Missing values BEFORE imputation
|
|
259
|
+
plt.clf()
|
|
260
|
+
ax = fig.add_subplot(111)
|
|
261
|
+
missing_before = self.data.isnull().sum()
|
|
262
|
+
missing_before = missing_before[missing_before > 0]
|
|
263
|
+
if not missing_before.empty:
|
|
264
|
+
missing_before.plot(kind="bar", color="crimson", edgecolor="black", ax=ax)
|
|
265
|
+
ax.set_title("Step 1: Missing Values (BEFORE Imputation)", fontsize=13, fontweight="bold")
|
|
266
|
+
ax.set_ylabel("Count of NaNs")
|
|
267
|
+
plt.xticks(rotation=45, ha="right")
|
|
268
|
+
plt.tight_layout()
|
|
269
|
+
plt.draw()
|
|
270
|
+
plt.pause(pause_time)
|
|
271
|
+
|
|
272
|
+
# Step 2: Impute Missing Values
|
|
273
|
+
if numeric_string_cols:
|
|
274
|
+
self.cleaner.clean_numeric_strings(numeric_string_cols)
|
|
275
|
+
self.cleaner.Clean_string(data_time=data_time)
|
|
276
|
+
self.cleaner.fill_missing(param=missing_param)
|
|
277
|
+
self.cleaner.fill_categorical_missing()
|
|
278
|
+
self._sync()
|
|
279
|
+
|
|
280
|
+
plt.clf()
|
|
281
|
+
ax = fig.add_subplot(111)
|
|
282
|
+
missing_after = self.data.isnull().sum()
|
|
283
|
+
missing_after.plot(kind="bar", color="mediumseagreen", edgecolor="black", ax=ax)
|
|
284
|
+
ax.set_title("Step 2: Missing Values (AFTER Imputation - All Filled)", fontsize=13, fontweight="bold")
|
|
285
|
+
ax.set_ylabel("Count of NaNs")
|
|
286
|
+
plt.xticks(rotation=45, ha="right")
|
|
287
|
+
plt.tight_layout()
|
|
288
|
+
plt.draw()
|
|
289
|
+
plt.pause(pause_time)
|
|
290
|
+
|
|
291
|
+
# Step 3: Outliers BEFORE Handling
|
|
292
|
+
num_cols = self.data.select_dtypes(include="number").columns.tolist()
|
|
293
|
+
if num_cols:
|
|
294
|
+
plt.clf()
|
|
295
|
+
ax = fig.add_subplot(111)
|
|
296
|
+
self.data[num_cols].boxplot(ax=ax, patch_artist=True, boxprops=dict(facecolor="orange"))
|
|
297
|
+
ax.set_title(f"Step 3: Outliers (BEFORE {outlier_method})", fontsize=13, fontweight="bold")
|
|
298
|
+
plt.xticks(rotation=45, ha="right")
|
|
299
|
+
plt.tight_layout()
|
|
300
|
+
plt.draw()
|
|
301
|
+
plt.pause(pause_time)
|
|
302
|
+
|
|
303
|
+
# Execute outlier handling
|
|
304
|
+
if outlier_method == "IQR":
|
|
305
|
+
self.cleaner.outliers_IQR(param=outlier_param)
|
|
306
|
+
else:
|
|
307
|
+
self.cleaner.outliers_Z(param=outlier_param)
|
|
308
|
+
self._sync()
|
|
309
|
+
|
|
310
|
+
# Step 4: Outliers AFTER Handling
|
|
311
|
+
plt.clf()
|
|
312
|
+
ax = fig.add_subplot(111)
|
|
313
|
+
self.data[num_cols].boxplot(ax=ax, patch_artist=True, boxprops=dict(facecolor="lightgreen"))
|
|
314
|
+
ax.set_title(f"Step 4: Outliers (AFTER {outlier_method} param='{outlier_param}')", fontsize=13, fontweight="bold")
|
|
315
|
+
plt.xticks(rotation=45, ha="right")
|
|
316
|
+
plt.tight_layout()
|
|
317
|
+
plt.draw()
|
|
318
|
+
plt.pause(pause_time)
|
|
319
|
+
|
|
320
|
+
# Step 5: Correlation Heatmap BEFORE low_corr
|
|
321
|
+
num_df = self.data.select_dtypes(include="number")
|
|
322
|
+
if num_df.shape[1] >= 2:
|
|
323
|
+
plt.clf()
|
|
324
|
+
ax = fig.add_subplot(111)
|
|
325
|
+
corr = num_df.corr()
|
|
326
|
+
cax = ax.imshow(corr, cmap="coolwarm", vmin=-1, vmax=1)
|
|
327
|
+
fig.colorbar(cax, ax=ax)
|
|
328
|
+
ax.set_xticks(range(len(corr.columns)))
|
|
329
|
+
ax.set_yticks(range(len(corr.columns)))
|
|
330
|
+
ax.set_xticklabels(corr.columns, rotation=45, ha="right")
|
|
331
|
+
ax.set_yticklabels(corr.columns)
|
|
332
|
+
ax.set_title("Step 5: Correlation Matrix (BEFORE low_corr filter)", fontsize=13, fontweight="bold")
|
|
333
|
+
plt.tight_layout()
|
|
334
|
+
plt.draw()
|
|
335
|
+
plt.pause(pause_time)
|
|
336
|
+
|
|
337
|
+
# Execute low_corr
|
|
338
|
+
if corr_threshold is not None:
|
|
339
|
+
self.cleaner.low_corr(target=corr_target, threshold=corr_threshold)
|
|
340
|
+
self._sync()
|
|
341
|
+
|
|
342
|
+
# Step 6: Correlation Heatmap AFTER low_corr
|
|
343
|
+
plt.clf()
|
|
344
|
+
ax = fig.add_subplot(111)
|
|
345
|
+
corr_after = self.data.select_dtypes(include="number").corr()
|
|
346
|
+
cax2 = ax.imshow(corr_after, cmap="coolwarm", vmin=-1, vmax=1)
|
|
347
|
+
fig.colorbar(cax2, ax=ax)
|
|
348
|
+
ax.set_xticks(range(len(corr_after.columns)))
|
|
349
|
+
ax.set_yticks(range(len(corr_after.columns)))
|
|
350
|
+
ax.set_xticklabels(corr_after.columns, rotation=45, ha="right")
|
|
351
|
+
ax.set_yticklabels(corr_after.columns)
|
|
352
|
+
ax.set_title("Step 6: Correlation Matrix (AFTER low_corr filter)", fontsize=13, fontweight="bold")
|
|
353
|
+
plt.tight_layout()
|
|
354
|
+
plt.draw()
|
|
355
|
+
plt.pause(pause_time)
|
|
356
|
+
|
|
357
|
+
logger.info("Live visual cleaning session completed")
|
|
358
|
+
except Exception as e:
|
|
359
|
+
logger.error("Error in live_clean: %s", e)
|
|
360
|
+
finally:
|
|
361
|
+
plt.ioff()
|
|
362
|
+
plt.show()
|
serverlex/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
_parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
5
|
+
if _parent_dir not in sys.path:
|
|
6
|
+
sys.path.insert(0, _parent_dir)
|
|
7
|
+
|
|
8
|
+
from Clean import Clean
|
|
9
|
+
from Visualization import Visualize
|
|
10
|
+
from DataCheck import DataGuard
|
|
11
|
+
from Pipeline import MLPipeline
|
|
12
|
+
from FeatureSelector import FeatureSelector
|
|
13
|
+
from ModelHyperparameter import ModelHyperparameter
|
|
14
|
+
from EnsembleBuilder import EnsembleBuilder
|
|
15
|
+
from Evaluation import ModelEvaluator
|
|
16
|
+
from Explainability import ModelExplainer
|
|
17
|
+
from ModelDeployer import ModelDeployer
|
|
18
|
+
from DriftDetector import DriftDetector
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"Clean",
|
|
22
|
+
"Visualize",
|
|
23
|
+
"DataGuard",
|
|
24
|
+
"MLPipeline",
|
|
25
|
+
"FeatureSelector",
|
|
26
|
+
"ModelHyperparameter",
|
|
27
|
+
"EnsembleBuilder",
|
|
28
|
+
"ModelEvaluator",
|
|
29
|
+
"ModelExplainer",
|
|
30
|
+
"ModelDeployer",
|
|
31
|
+
"DriftDetector"
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
__version__ = "1.0.0"
|