serverlex 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Pipeline.py ADDED
@@ -0,0 +1,251 @@
1
+ import logging
2
+ import pandas as pd
3
+ import numpy as np
4
+ from Clean import Clean
5
+
6
+ from sklearn.model_selection import cross_val_score, KFold, StratifiedKFold
7
+ from sklearn.linear_model import LogisticRegression, Ridge
8
+ from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
9
+ from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor, GradientBoostingClassifier, GradientBoostingRegressor
10
+
11
+ logger=logging.getLogger("Pipeline")
12
+ if not logger.handlers:
13
+ _h=logging.StreamHandler()
14
+ _h.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", "%Y-%m-%d %H:%M:%S"))
15
+ logger.addHandler(_h)
16
+ logger.setLevel(logging.INFO)
17
+
18
+
19
+ class MLPipeline:
20
+ def __init__(self, data=None):
21
+ self.data=None
22
+ self.cleaner=None
23
+ self.target=None
24
+ self.feature_columns=[]
25
+ self.num_fill={}
26
+ self.cat_fill={}
27
+ self.scaler_mean={}
28
+ self.scaler_std={}
29
+ self.date_col=None
30
+ self.custom_models={}
31
+
32
+ if data is not None:
33
+ self.load_data(data)
34
+
35
+ def load_data(self, data):
36
+ if isinstance(data, str):
37
+ self.cleaner=Clean(data)
38
+ self.cleaner.load()
39
+ self.data=self.cleaner.data
40
+ elif isinstance(data, Clean):
41
+ self.cleaner=data
42
+ self.data=self.cleaner.data
43
+ elif isinstance(data, pd.DataFrame):
44
+ self.cleaner=Clean.__new__(Clean)
45
+ self.cleaner.data=data.copy()
46
+ self.cleaner.path=None
47
+ self.data=data
48
+
49
+ # ==========================
50
+ # 1. FIT (Learn & Save on self)
51
+ # ==========================
52
+ def fit(self, df, target=None, encode='onehot', scale='standard', data_time=None, numeric_string_cols=None, **kwargs):
53
+ try:
54
+ self.load_data(df)
55
+ self.target=target.strip().lower() if target else None
56
+ self.encode=encode
57
+ self.scale=scale
58
+ self.numeric_string_cols=[c.strip().lower().replace(' ', '_') for c in (numeric_string_cols or [])]
59
+
60
+ logger.info("Fitting pipeline on %d rows (target=%s, encode=%s, scale=%s)", len(self.data), self.target, self.encode, self.scale)
61
+
62
+ # 1. Clean the training data using your Clean class
63
+ self.cleaner.total_clean(data_time=data_time, numeric_string_cols=numeric_string_cols, **kwargs)
64
+ self.data=self.cleaner.data
65
+ self.cleaner.seperate()
66
+
67
+ # 2. Extract datetime features if provided
68
+ if data_time:
69
+ col=data_time[0].strip().lower().replace(' ', '_')
70
+ if col in self.data.columns:
71
+ self.date_col=col
72
+ dt=pd.to_datetime(self.data[col], format='mixed', errors='coerce')
73
+ self.data[f'{col}_year']=dt.dt.year
74
+ self.data[f'{col}_month']=dt.dt.month
75
+ self.data[f'{col}_day']=dt.dt.day
76
+ self.data[f'{col}_dayofweek']=dt.dt.dayofweek
77
+ self.data=self.data.drop(columns=[col])
78
+
79
+ self.cleaner.data=self.data
80
+ self.cleaner.seperate()
81
+
82
+ # 3. Memorize missing value fill values directly on self
83
+ for col in self.cleaner.numerical_columns.columns:
84
+ if col != self.target:
85
+ self.num_fill[col]=self.data[col].mean()
86
+
87
+ for col in self.cleaner.categrorical_columns.columns:
88
+ if col != self.target:
89
+ mode_val=self.data[col].mode()
90
+ self.cat_fill[col]=mode_val[0] if not mode_val.empty else 'unknown'
91
+
92
+ # 4. Encode categoricals to find final feature columns
93
+ train_X=self.data.copy()
94
+ if self.target and self.target in train_X.columns:
95
+ train_X=train_X.drop(columns=[self.target])
96
+
97
+ cat_cols=[c for c in self.cleaner.categrorical_columns.columns if c != self.target and c in train_X.columns]
98
+ if self.encode=='onehot' and cat_cols:
99
+ train_X=pd.get_dummies(train_X, columns=cat_cols, drop_first=True, dtype=int)
100
+ elif self.encode=='label' and cat_cols:
101
+ for col in cat_cols:
102
+ train_X[col]=train_X[col].astype('category').cat.codes
103
+
104
+ # Save exact list of final columns for future test data
105
+ self.feature_columns=train_X.columns.tolist()
106
+
107
+ # 5. Memorize scaling numbers on self
108
+ if self.scale=='standard':
109
+ for col in self.cleaner.numerical_columns.columns:
110
+ if col in self.feature_columns:
111
+ self.scaler_mean[col]=train_X[col].mean()
112
+ std_val=train_X[col].std()
113
+ self.scaler_std[col]=std_val if std_val != 0 else 1.0
114
+
115
+ logger.info("Fit complete. Feature count: %d", len(self.feature_columns))
116
+ return self
117
+ except Exception as e:
118
+ logger.error("Error during fit: %s", e)
119
+ return self
120
+
121
+ # ==========================
122
+ # 2. TRANSFORM (Uses self variables)
123
+ # ==========================
124
+ def transform(self, df):
125
+ try:
126
+ out=df.copy()
127
+
128
+ # 1. Clean column names
129
+ out.columns=out.columns.str.strip().str.lower().str.replace(' ', '_').str.replace(r'[^\w_]', '', regex=True)
130
+
131
+ # Clean numeric string columns using saved self.numeric_string_cols
132
+ if hasattr(self, 'numeric_string_cols'):
133
+ for col in self.numeric_string_cols:
134
+ if col in out.columns:
135
+ cleaned=out[col].astype(str).str.replace(r'[\$,]', '', regex=True)
136
+ out[col]=pd.to_numeric(cleaned, errors='coerce')
137
+
138
+ # 2. Extract datetime features using saved date_col
139
+ if self.date_col and self.date_col in out.columns:
140
+ dt=pd.to_datetime(out[self.date_col], format='mixed', errors='coerce')
141
+ out[f'{self.date_col}_year']=dt.dt.year
142
+ out[f'{self.date_col}_month']=dt.dt.month
143
+ out[f'{self.date_col}_day']=dt.dt.day
144
+ out[f'{self.date_col}_dayofweek']=dt.dt.dayofweek
145
+ out=out.drop(columns=[self.date_col])
146
+
147
+ # 3. Fill missing using memorized self.num_fill and self.cat_fill
148
+ for col, val in self.num_fill.items():
149
+ if col in out.columns:
150
+ out[col]=out[col].fillna(val)
151
+
152
+ for col, val in self.cat_fill.items():
153
+ if col in out.columns:
154
+ out[col]=out[col].astype(str).str.lower().str.replace(' ', '')
155
+ out[col]=out[col].fillna(val)
156
+
157
+ # 4. Check target
158
+ has_target=self.target and self.target in out.columns
159
+ target_series=out[self.target] if has_target else None
160
+
161
+ # 5. Encode categoricals
162
+ cat_cols=[c for c in self.cat_fill.keys() if c in out.columns]
163
+ if self.encode=='onehot' and cat_cols:
164
+ out=pd.get_dummies(out, columns=cat_cols, drop_first=True, dtype=int)
165
+ elif self.encode=='label' and cat_cols:
166
+ for col in cat_cols:
167
+ out[col]=out[col].astype('category').cat.codes
168
+
169
+ # 6. Reindex columns so test columns match self.feature_columns exactly
170
+ X=out.reindex(columns=self.feature_columns, fill_value=0)
171
+
172
+ # 7. Scale using memorized self.scaler_mean and self.scaler_std
173
+ if self.scale=='standard':
174
+ for col in self.scaler_mean.keys():
175
+ if col in X.columns:
176
+ X[col]=(X[col] - self.scaler_mean[col]) / self.scaler_std[col]
177
+
178
+ if has_target:
179
+ return X, target_series
180
+ return X
181
+ except Exception as e:
182
+ logger.error("Error during transform: %s", e)
183
+ return df
184
+
185
+ def fit_transform(self, df, **kwargs):
186
+ self.fit(df, **kwargs)
187
+ return self.transform(df)
188
+
189
+ # ==========================
190
+ # 3. Custom Models & Evaluation
191
+ # ==========================
192
+ def add_model(self, name, model):
193
+ self.custom_models[name]=model
194
+ logger.info("Registered custom model '%s'", name)
195
+ return self
196
+
197
+ def evaluate_baseline_models(self, X, y, cv=5):
198
+ try:
199
+ if not isinstance(y, pd.Series):
200
+ y=pd.Series(y)
201
+
202
+ is_classification=y.dtype=='object' or str(y.dtype).startswith('cat') or y.nunique() <= 20
203
+ task="Classification" if is_classification else "Regression"
204
+
205
+ results=[]
206
+
207
+ if is_classification:
208
+ kf=StratifiedKFold(n_splits=cv, shuffle=True, random_state=42)
209
+ models={
210
+ "Logistic Regression": LogisticRegression(max_iter=1000),
211
+ "Decision Tree": DecisionTreeClassifier(random_state=42),
212
+ "Random Forest": RandomForestClassifier(n_estimators=100, random_state=42),
213
+ "Gradient Boosting": GradientBoostingClassifier(random_state=42)
214
+ }
215
+ models.update(self.custom_models)
216
+ logger.info("Evaluating %d baseline models (task=%s, cv=%d)", len(models), task, cv)
217
+
218
+ for name, model in models.items():
219
+ try:
220
+ acc=cross_val_score(model, X, y, cv=kf, scoring="accuracy")
221
+ f1=cross_val_score(model, X, y, cv=kf, scoring="f1_weighted")
222
+ results.append({"Model": name, "Accuracy": round(acc.mean(), 4), "F1": round(f1.mean(), 4)})
223
+ except Exception as e:
224
+ results.append({"Model": name, "Accuracy": None, "F1": str(e)})
225
+
226
+ leaderboard=pd.DataFrame(results).sort_values(by="Accuracy", ascending=False)
227
+ else:
228
+ kf=KFold(n_splits=cv, shuffle=True, random_state=42)
229
+ models={
230
+ "Ridge Regression": Ridge(),
231
+ "Decision Tree": DecisionTreeRegressor(random_state=42),
232
+ "Random Forest": RandomForestRegressor(n_estimators=100, random_state=42),
233
+ "Gradient Boosting": GradientBoostingRegressor(random_state=42)
234
+ }
235
+ models.update(self.custom_models)
236
+ logger.info("Evaluating %d baseline models (task=%s, cv=%d)", len(models), task, cv)
237
+
238
+ for name, model in models.items():
239
+ try:
240
+ r2=cross_val_score(model, X, y, cv=kf, scoring="r2")
241
+ rmse=cross_val_score(model, X, y, cv=kf, scoring="neg_root_mean_squared_error")
242
+ results.append({"Model": name, "R2": round(r2.mean(), 4), "RMSE": round(-rmse.mean(), 4)})
243
+ except Exception as e:
244
+ results.append({"Model": name, "R2": None, "RMSE": str(e)})
245
+
246
+ leaderboard=pd.DataFrame(results).sort_values(by="R2", ascending=False)
247
+
248
+ logger.info("Baseline leaderboard:\n%s", leaderboard.to_string(index=False))
249
+ return leaderboard
250
+ except Exception as e:
251
+ logger.error("Error in evaluate_baseline_models: %s", e)
Visualization.py ADDED
@@ -0,0 +1,362 @@
1
+ import logging
2
+ import matplotlib.pyplot as plt
3
+ import pandas as pd
4
+ import numpy as np
5
+ from Clean import Clean
6
+
7
+ logger=logging.getLogger("Visualization")
8
+ if not logger.handlers:
9
+ _h=logging.StreamHandler()
10
+ _h.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", "%Y-%m-%d %H:%M:%S"))
11
+ logger.addHandler(_h)
12
+ logger.setLevel(logging.INFO)
13
+
14
+
15
+ class Visualize:
16
+ def __init__(self, data=None):
17
+ """
18
+ data can be:
19
+ - A CSV file path (str)
20
+ - An instance of Clean
21
+ - A pandas DataFrame
22
+ - None (data can be set or loaded later)
23
+ """
24
+ self.data = None
25
+ self.cleaner = None
26
+ if data is not None:
27
+ self.set_data(data)
28
+
29
+ def set_data(self, data):
30
+ """Sets or updates the active dataset."""
31
+ if isinstance(data, str):
32
+ self.cleaner = Clean(data)
33
+ self.cleaner.load()
34
+ self.data = self.cleaner.data
35
+ elif isinstance(data, Clean):
36
+ self.cleaner = data
37
+ if not hasattr(self.cleaner, "data") or self.cleaner.data is None:
38
+ self.cleaner.load()
39
+ self.data = self.cleaner.data
40
+ elif isinstance(data, pd.DataFrame):
41
+ self.cleaner = Clean.__new__(Clean)
42
+ self.cleaner.data = data.copy()
43
+ self.cleaner.path = None
44
+ self.data = data
45
+
46
+ def _sync(self):
47
+ """Syncs self.data from self.cleaner if cleaner modified the dataframe."""
48
+ if self.cleaner is not None and hasattr(self.cleaner, "data") and self.cleaner.data is not None:
49
+ self.data = self.cleaner.data
50
+
51
+ # ==========================
52
+ # 1. Individual Plot Methods
53
+ # ==========================
54
+
55
+ def plot_missing(self, title="Missing Values per Column"):
56
+ """Plots a bar chart showing the count of missing values for columns with NaNs."""
57
+ self._sync()
58
+ try:
59
+ if self.data is None:
60
+ print("No data available to plot.")
61
+ return
62
+
63
+ missing = self.data.isnull().sum()
64
+ missing = missing[missing > 0]
65
+ if missing.empty:
66
+ print("No missing values found in dataset.")
67
+ return
68
+
69
+ fig, ax = plt.subplots(figsize=(8, 4))
70
+ missing.plot(kind="bar", color="salmon", edgecolor="black", ax=ax)
71
+ ax.set_title(title, fontsize=12, fontweight="bold")
72
+ ax.set_ylabel("Count of NaNs")
73
+ ax.set_xlabel("Columns")
74
+ plt.xticks(rotation=45, ha="right")
75
+ plt.tight_layout()
76
+ plt.show()
77
+ except Exception as e:
78
+ print(e)
79
+
80
+ def plot_distribution(self, column=None, bins=20):
81
+ """Plots histograms for one or all numerical columns."""
82
+ self._sync()
83
+ try:
84
+ if self.data is None:
85
+ print("No data available to plot.")
86
+ return
87
+
88
+ num_cols = self.data.select_dtypes(include="number").columns
89
+ if len(num_cols) == 0:
90
+ print("No numerical columns available.")
91
+ return
92
+
93
+ cols = [column] if column else num_cols
94
+ n_cols = len(cols)
95
+ fig, axes = plt.subplots(1, n_cols, figsize=(4 * n_cols, 4), squeeze=False)
96
+ for i, col in enumerate(cols):
97
+ if col in self.data.columns:
98
+ axes[0, i].hist(self.data[col].dropna(), bins=bins, color="skyblue", edgecolor="black")
99
+ axes[0, i].set_title(f"Dist: {col}")
100
+ axes[0, i].set_xlabel(col)
101
+ axes[0, i].set_ylabel("Frequency")
102
+ plt.tight_layout()
103
+ plt.show()
104
+ except Exception as e:
105
+ print(e)
106
+
107
+ def plot_box(self, columns=None, title="Outlier Detection (Box Plot)"):
108
+ """Plots box plots for numeric columns to visualize outliers."""
109
+ self._sync()
110
+ try:
111
+ if self.data is None:
112
+ print("No data available to plot.")
113
+ return
114
+
115
+ num_df = self.data.select_dtypes(include="number")
116
+ if num_df.empty:
117
+ print("No numerical columns for box plot.")
118
+ return
119
+
120
+ if columns:
121
+ valid_cols = [c for c in columns if c in num_df.columns]
122
+ num_df = num_df[valid_cols]
123
+
124
+ fig, ax = plt.subplots(figsize=(max(6, len(num_df.columns) * 1.5), 5))
125
+ num_df.boxplot(ax=ax, patch_artist=True, boxprops=dict(facecolor="lightblue"))
126
+ ax.set_title(title, fontsize=12, fontweight="bold")
127
+ plt.xticks(rotation=45, ha="right")
128
+ plt.tight_layout()
129
+ plt.show()
130
+ except Exception as e:
131
+ print(e)
132
+
133
+ def plot_categorical(self, column=None, top_n=10, title=None):
134
+ """Plots horizontal bar chart of top N categories."""
135
+ self._sync()
136
+ try:
137
+ if self.data is None:
138
+ print("No data available to plot.")
139
+ return
140
+
141
+ cat_cols = self.data.select_dtypes(include=["object", "category"]).columns
142
+ if len(cat_cols) == 0:
143
+ print("No categorical columns available.")
144
+ return
145
+
146
+ col = column if column else cat_cols[0]
147
+ if col not in self.data.columns:
148
+ print(f"Column '{col}' not found.")
149
+ return
150
+
151
+ counts = self.data[col].value_counts().head(top_n)
152
+ fig, ax = plt.subplots(figsize=(8, 4))
153
+ counts.plot(kind="barh", color="mediumaquamarine", edgecolor="black", ax=ax)
154
+ ax.invert_yaxis()
155
+ ax.set_title(title or f"Top {top_n} Categories in '{col}'", fontsize=12, fontweight="bold")
156
+ ax.set_xlabel("Count")
157
+ plt.tight_layout()
158
+ plt.show()
159
+ except Exception as e:
160
+ print(e)
161
+
162
+ def plot_correlation(self, title="Correlation Matrix Heatmap"):
163
+ """Plots a correlation heatmap for numerical features using matplotlib."""
164
+ self._sync()
165
+ try:
166
+ if self.data is None:
167
+ print("No data available to plot.")
168
+ return
169
+
170
+ num_df = self.data.select_dtypes(include="number")
171
+ if num_df.shape[1] < 2:
172
+ print("Need at least 2 numerical columns for correlation plot.")
173
+ return
174
+
175
+ corr = num_df.corr()
176
+ fig, ax = plt.subplots(figsize=(max(6, len(corr) * 1.2), max(5, len(corr) * 1.0)))
177
+ cax = ax.imshow(corr, cmap="coolwarm", vmin=-1, vmax=1)
178
+ fig.colorbar(cax, ax=ax)
179
+
180
+ ax.set_xticks(range(len(corr.columns)))
181
+ ax.set_yticks(range(len(corr.columns)))
182
+ ax.set_xticklabels(corr.columns, rotation=45, ha="right")
183
+ ax.set_yticklabels(corr.columns)
184
+
185
+ # Annotate correlation values in cells
186
+ for i in range(len(corr)):
187
+ for j in range(len(corr.columns)):
188
+ val = corr.iloc[i, j]
189
+ ax.text(j, i, f"{val:.2f}", ha="center", va="center", color="black" if abs(val) < 0.7 else "white", fontsize=9)
190
+
191
+ ax.set_title(title, fontsize=12, fontweight="bold")
192
+ plt.tight_layout()
193
+ plt.show()
194
+ except Exception as e:
195
+ print(e)
196
+
197
+ def plot_scatter(self, x, y, title=None):
198
+ """Plots a scatter plot between two numerical columns."""
199
+ self._sync()
200
+ try:
201
+ if self.data is None:
202
+ print("No data available to plot.")
203
+ return
204
+
205
+ if x not in self.data.columns or y not in self.data.columns:
206
+ print(f"One or both columns ({x}, {y}) not in DataFrame.")
207
+ return
208
+
209
+ fig, ax = plt.subplots(figsize=(6, 4))
210
+ ax.scatter(self.data[x], self.data[y], color="steelblue", alpha=0.7, edgecolors="k")
211
+ ax.set_title(title or f"{x} vs {y}", fontsize=12, fontweight="bold")
212
+ ax.set_xlabel(x)
213
+ ax.set_ylabel(y)
214
+ ax.grid(True, linestyle="--", alpha=0.5)
215
+ plt.tight_layout()
216
+ plt.show()
217
+ except Exception as e:
218
+ print(e)
219
+
220
+ # ==============================================================
221
+ # 2. Live Interactive Cleaning Pipeline with plt.ion()
222
+ # ==============================================================
223
+
224
+ def live_clean(
225
+ self,
226
+ cleaner=None,
227
+ data_time=None,
228
+ numeric_string_cols=None,
229
+ missing_param="m",
230
+ outlier_method="IQR",
231
+ outlier_param="m",
232
+ corr_target=None,
233
+ corr_threshold=0.1,
234
+ pause_time=1.5
235
+ ):
236
+ """
237
+ Runs the cleaning pipeline step-by-step with plt.ion() (interactive mode),
238
+ updating live plots at each stage to show before & after transformations.
239
+ """
240
+ if cleaner is not None:
241
+ self.cleaner = cleaner
242
+ elif self.cleaner is None:
243
+ print("No cleaner instance provided.")
244
+ return
245
+
246
+ # Enable interactive plotting
247
+ plt.ion()
248
+ fig = plt.figure(figsize=(10, 6))
249
+ logger.info("Initializing live visual cleaning session")
250
+
251
+ try:
252
+ # 1. Load & Deduplicate
253
+ self.cleaner.load()
254
+ self.cleaner.clean_column_names()
255
+ self.cleaner.delete_duplicates()
256
+ self._sync()
257
+
258
+ # Step 1: Missing values BEFORE imputation
259
+ plt.clf()
260
+ ax = fig.add_subplot(111)
261
+ missing_before = self.data.isnull().sum()
262
+ missing_before = missing_before[missing_before > 0]
263
+ if not missing_before.empty:
264
+ missing_before.plot(kind="bar", color="crimson", edgecolor="black", ax=ax)
265
+ ax.set_title("Step 1: Missing Values (BEFORE Imputation)", fontsize=13, fontweight="bold")
266
+ ax.set_ylabel("Count of NaNs")
267
+ plt.xticks(rotation=45, ha="right")
268
+ plt.tight_layout()
269
+ plt.draw()
270
+ plt.pause(pause_time)
271
+
272
+ # Step 2: Impute Missing Values
273
+ if numeric_string_cols:
274
+ self.cleaner.clean_numeric_strings(numeric_string_cols)
275
+ self.cleaner.Clean_string(data_time=data_time)
276
+ self.cleaner.fill_missing(param=missing_param)
277
+ self.cleaner.fill_categorical_missing()
278
+ self._sync()
279
+
280
+ plt.clf()
281
+ ax = fig.add_subplot(111)
282
+ missing_after = self.data.isnull().sum()
283
+ missing_after.plot(kind="bar", color="mediumseagreen", edgecolor="black", ax=ax)
284
+ ax.set_title("Step 2: Missing Values (AFTER Imputation - All Filled)", fontsize=13, fontweight="bold")
285
+ ax.set_ylabel("Count of NaNs")
286
+ plt.xticks(rotation=45, ha="right")
287
+ plt.tight_layout()
288
+ plt.draw()
289
+ plt.pause(pause_time)
290
+
291
+ # Step 3: Outliers BEFORE Handling
292
+ num_cols = self.data.select_dtypes(include="number").columns.tolist()
293
+ if num_cols:
294
+ plt.clf()
295
+ ax = fig.add_subplot(111)
296
+ self.data[num_cols].boxplot(ax=ax, patch_artist=True, boxprops=dict(facecolor="orange"))
297
+ ax.set_title(f"Step 3: Outliers (BEFORE {outlier_method})", fontsize=13, fontweight="bold")
298
+ plt.xticks(rotation=45, ha="right")
299
+ plt.tight_layout()
300
+ plt.draw()
301
+ plt.pause(pause_time)
302
+
303
+ # Execute outlier handling
304
+ if outlier_method == "IQR":
305
+ self.cleaner.outliers_IQR(param=outlier_param)
306
+ else:
307
+ self.cleaner.outliers_Z(param=outlier_param)
308
+ self._sync()
309
+
310
+ # Step 4: Outliers AFTER Handling
311
+ plt.clf()
312
+ ax = fig.add_subplot(111)
313
+ self.data[num_cols].boxplot(ax=ax, patch_artist=True, boxprops=dict(facecolor="lightgreen"))
314
+ ax.set_title(f"Step 4: Outliers (AFTER {outlier_method} param='{outlier_param}')", fontsize=13, fontweight="bold")
315
+ plt.xticks(rotation=45, ha="right")
316
+ plt.tight_layout()
317
+ plt.draw()
318
+ plt.pause(pause_time)
319
+
320
+ # Step 5: Correlation Heatmap BEFORE low_corr
321
+ num_df = self.data.select_dtypes(include="number")
322
+ if num_df.shape[1] >= 2:
323
+ plt.clf()
324
+ ax = fig.add_subplot(111)
325
+ corr = num_df.corr()
326
+ cax = ax.imshow(corr, cmap="coolwarm", vmin=-1, vmax=1)
327
+ fig.colorbar(cax, ax=ax)
328
+ ax.set_xticks(range(len(corr.columns)))
329
+ ax.set_yticks(range(len(corr.columns)))
330
+ ax.set_xticklabels(corr.columns, rotation=45, ha="right")
331
+ ax.set_yticklabels(corr.columns)
332
+ ax.set_title("Step 5: Correlation Matrix (BEFORE low_corr filter)", fontsize=13, fontweight="bold")
333
+ plt.tight_layout()
334
+ plt.draw()
335
+ plt.pause(pause_time)
336
+
337
+ # Execute low_corr
338
+ if corr_threshold is not None:
339
+ self.cleaner.low_corr(target=corr_target, threshold=corr_threshold)
340
+ self._sync()
341
+
342
+ # Step 6: Correlation Heatmap AFTER low_corr
343
+ plt.clf()
344
+ ax = fig.add_subplot(111)
345
+ corr_after = self.data.select_dtypes(include="number").corr()
346
+ cax2 = ax.imshow(corr_after, cmap="coolwarm", vmin=-1, vmax=1)
347
+ fig.colorbar(cax2, ax=ax)
348
+ ax.set_xticks(range(len(corr_after.columns)))
349
+ ax.set_yticks(range(len(corr_after.columns)))
350
+ ax.set_xticklabels(corr_after.columns, rotation=45, ha="right")
351
+ ax.set_yticklabels(corr_after.columns)
352
+ ax.set_title("Step 6: Correlation Matrix (AFTER low_corr filter)", fontsize=13, fontweight="bold")
353
+ plt.tight_layout()
354
+ plt.draw()
355
+ plt.pause(pause_time)
356
+
357
+ logger.info("Live visual cleaning session completed")
358
+ except Exception as e:
359
+ logger.error("Error in live_clean: %s", e)
360
+ finally:
361
+ plt.ioff()
362
+ plt.show()
serverlex/__init__.py ADDED
@@ -0,0 +1,34 @@
1
+ import sys
2
+ import os
3
+
4
+ _parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
5
+ if _parent_dir not in sys.path:
6
+ sys.path.insert(0, _parent_dir)
7
+
8
+ from Clean import Clean
9
+ from Visualization import Visualize
10
+ from DataCheck import DataGuard
11
+ from Pipeline import MLPipeline
12
+ from FeatureSelector import FeatureSelector
13
+ from ModelHyperparameter import ModelHyperparameter
14
+ from EnsembleBuilder import EnsembleBuilder
15
+ from Evaluation import ModelEvaluator
16
+ from Explainability import ModelExplainer
17
+ from ModelDeployer import ModelDeployer
18
+ from DriftDetector import DriftDetector
19
+
20
+ __all__ = [
21
+ "Clean",
22
+ "Visualize",
23
+ "DataGuard",
24
+ "MLPipeline",
25
+ "FeatureSelector",
26
+ "ModelHyperparameter",
27
+ "EnsembleBuilder",
28
+ "ModelEvaluator",
29
+ "ModelExplainer",
30
+ "ModelDeployer",
31
+ "DriftDetector"
32
+ ]
33
+
34
+ __version__ = "1.0.0"