serverlex 1.2.0__tar.gz → 1.2.1__tar.gz

{serverlex-1.2.0 → serverlex-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: serverlex
-Version: 1.2.0
+Version: 1.2.1
 Summary: Production-Safe ML Preprocessing, Diagnostics, Tuning, and Serving Platform
 Author: Serverlex Team
 License: MIT

{serverlex-1.2.0 → serverlex-1.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "serverlex"
-version = "1.2.0"
+version = "1.2.1"
 description = "Production-Safe ML Preprocessing, Diagnostics, Tuning, and Serving Platform"
 readme = "README.md"
 authors = [{ name = "Serverlex Team" }]

{serverlex-1.2.0 → serverlex-1.2.1}/serverlex/__init__.py

@@ -24,4 +24,4 @@ __all__ = [
     "ModelDeployer",
 ]
 
-__version__ = "1.2.0"
+__version__ = "1.2.1"

{serverlex-1.2.0 → serverlex-1.2.1}/serverlex/diagnostics/drift.py

@@ -87,19 +87,46 @@ class DriftDetector:
             logger.info("Running drift detection across %d features (alpha=%.3f, psi_threshold=%.2f)", len(ref_cols), self.alpha, self.psi_threshold)
 
             for col in common_cols:
-                # 2. Numerical Drift
-                if pd.api.types.is_numeric_dtype(self.ref_data[col]):
-                    ref_vals=self.ref_data[col].dropna()
-                    cur_vals=cur_data[col].dropna()
+                ref_series = self.ref_data[col].dropna()
+                cur_series = cur_data[col].dropna()
+
+                if len(ref_series) == 0 or len(cur_series) == 0:
+                    continue
 
-                    if len(ref_vals) > 0 and len(cur_vals) > 0:
-                        stat, p_val = ks_2samp(ref_vals, cur_vals)
-                        psi = self.calculate_psi(ref_vals, cur_vals)
-                        w_dist = float(np.round(wasserstein_distance(ref_vals, cur_vals), 4))
+                # 2. Check Constant Shift (was constant in reference, altered in current)
+                if ref_series.nunique() <= 1:
+                    ref_val = ref_series.iloc[0] if len(ref_series) > 0 else None
+                    is_shifted = (cur_series.nunique() > 1) or (cur_series.nunique() == 1 and cur_series.iloc[0] != ref_val)
+                    if is_shifted:
+                        drift_count += 1
+                        logger.warning("Constant feature shift detected in '%s' (ref=%s, cur_unique=%s)", col, ref_val, cur_series.unique())
+                        self.drift_results.append({
+                            "Feature": col,
+                            "Type": "Constant Shift",
+                            "Statistic": None,
+                            "P_Value": 0.0,
+                            "PSI": 1.0,
+                            "Wasserstein": None,
+                            "Drift_Detected": True,
+                            "Severity": "High"
+                        })
+                        continue
 
-                        # Avoid false positives: require practical shift (PSI >= threshold) OR severe shift
+                # 3. Numerical Drift
+                if pd.api.types.is_numeric_dtype(self.ref_data[col]):
+                    ref_vals = ref_series
+                    cur_vals = cur_series
+
+                    stat, p_val = ks_2samp(ref_vals, cur_vals)
+                    psi = self.calculate_psi(ref_vals, cur_vals)
+                    w_dist = float(np.round(wasserstein_distance(ref_vals, cur_vals), 4))
+
+                    # Guard against small sample noise: KS p-value requires sample size >= 10
+                    if len(cur_vals) < 10:
+                        is_drift = False
+                        severity = "Insufficient Samples (< 10)"
+                    else:
                         is_drift = (psi >= self.psi_threshold) or (p_val < self.alpha and psi >= 0.10)
-                        
                         if psi >= 0.25:
                             severity = "High"
                         elif psi >= 0.10:
@@ -107,26 +134,40 @@ class DriftDetector:
                         else:
                             severity = "None"
 
-                        if is_drift:
-                            drift_count += 1
-                            logger.warning("Data drift detected in '%s' (KS=%.4f, p=%.5f, PSI=%.4f, Wasserstein=%.4f, Severity=%s)", col, stat, p_val, psi, w_dist, severity)
+                    if is_drift:
+                        drift_count += 1
+                        logger.warning("Data drift detected in '%s' (KS=%.4f, p=%.5f, PSI=%.4f, Wasserstein=%.4f, Severity=%s)", col, stat, p_val, psi, w_dist, severity)
 
-                        self.drift_results.append({
-                            "Feature": col,
-                            "Type": "Numerical",
-                            "Statistic": round(stat, 4),
-                            "P_Value": round(p_val, 5),
-                            "PSI": psi,
-                            "Wasserstein": w_dist,
-                            "Drift_Detected": is_drift,
-                            "Severity": severity
-                        })
+                    self.drift_results.append({
+                        "Feature": col,
+                        "Type": "Numerical",
+                        "Statistic": round(stat, 4),
+                        "P_Value": round(p_val, 5),
+                        "PSI": psi,
+                        "Wasserstein": w_dist,
+                        "Drift_Detected": is_drift,
+                        "Severity": severity
+                    })
 
-                # 3. Categorical Drift
+                # 4. Categorical Drift
                 else:
                     ref_counts=self.ref_data[col].astype(str).value_counts()
                     cur_counts=cur_data[col].astype(str).value_counts()
 
+                    # Guard against tiny batches: Chi-square and PSI are unstable for n < 10
+                    if len(cur_series) < 10:
+                        self.drift_results.append({
+                            "Feature": col,
+                            "Type": "Categorical",
+                            "Statistic": None,
+                            "P_Value": 1.0,
+                            "PSI": 0.0,
+                            "Wasserstein": None,
+                            "Drift_Detected": False,
+                            "Severity": "Insufficient Samples (< 10)"
+                        })
+                        continue
+
                     all_cats=list(set(ref_counts.index).union(set(cur_counts.index)))
                     ref_aligned=np.array([ref_counts.get(cat, 0) for cat in all_cats], dtype=float)
                     cur_aligned=np.array([cur_counts.get(cat, 0) for cat in all_cats], dtype=float)

{serverlex-1.2.0 → serverlex-1.2.1}/serverlex/diagnostics/evaluation.py

@@ -34,7 +34,9 @@ class ModelEvaluator:
     # ==============================================================
     # 1. Classification Diagnostics
     # ==============================================================
-    def plot_confusion_matrix(self, y_true, y_pred, labels=None, title="Confusion Matrix"):
+    # 1. Classification Diagnostics
+    # ==============================================================
+    def plot_confusion_matrix(self, y_true, y_pred, labels=None, title="Confusion Matrix", save_path=None):
         try:
             cm=confusion_matrix(y_true, y_pred, labels=labels)
             fig, ax=plt.subplots(figsize=(6, 5))
@@ -58,12 +60,17 @@ class ModelEvaluator:
             ax.set_ylabel("Actual Label")
             ax.set_xlabel("Predicted Label")
             plt.tight_layout()
-            plt.show()
+            if save_path:
+                plt.savefig(save_path)
+                plt.close(fig)
+            else:
+                plt.show()
             return cm
         except Exception as e:
             logger.error("Error in plot_confusion_matrix: %s", e)
+            raise
 
-    def plot_roc_curve(self, y_true, y_prob, title="ROC Curve"):
+    def plot_roc_curve(self, y_true, y_prob, title="ROC Curve", save_path=None):
         try:
             y_true_arr = np.asarray(y_true)
             y_prob_arr = np.asarray(y_prob)
@@ -98,14 +105,18 @@ class ModelEvaluator:
             ax.legend(loc="lower right")
             ax.grid(True, linestyle="--", alpha=0.5)
             plt.tight_layout()
-            plt.show()
+            if save_path:
+                plt.savefig(save_path)
+                plt.close(fig)
+            else:
+                plt.show()
             logger.info("ROC-AUC score: %.4f", roc_auc)
             return roc_auc
         except Exception as e:
             logger.error("Error in plot_roc_curve: %s", e)
             raise
 
-    def plot_precision_recall(self, y_true, y_prob, title="Precision-Recall Curve"):
+    def plot_precision_recall(self, y_true, y_prob, title="Precision-Recall Curve", save_path=None):
         try:
             precision, recall, _ = precision_recall_curve(y_true, y_prob)
             fig, ax=plt.subplots(figsize=(6, 5))
@@ -115,14 +126,19 @@ class ModelEvaluator:
             ax.set_title(title, fontsize=12, fontweight="bold")
             ax.grid(True, linestyle="--", alpha=0.5)
             plt.tight_layout()
-            plt.show()
+            if save_path:
+                plt.savefig(save_path)
+                plt.close(fig)
+            else:
+                plt.show()
         except Exception as e:
             logger.error("Error in plot_precision_recall: %s", e)
+            raise
 
     # ==============================================================
     # 2. Regression Diagnostics
     # ==============================================================
-    def plot_actual_vs_predicted(self, y_true, y_pred, title="Actual vs. Predicted"):
+    def plot_actual_vs_predicted(self, y_true, y_pred, title="Actual vs. Predicted", save_path=None):
         try:
             fig, ax=plt.subplots(figsize=(6, 5))
             ax.scatter(y_true, y_pred, alpha=0.6, edgecolors="k", color="royalblue")
@@ -137,11 +153,16 @@ class ModelEvaluator:
             ax.legend()
             ax.grid(True, linestyle="--", alpha=0.5)
             plt.tight_layout()
-            plt.show()
+            if save_path:
+                plt.savefig(save_path)
+                plt.close(fig)
+            else:
+                plt.show()
         except Exception as e:
             logger.error("Error in plot_actual_vs_predicted: %s", e)
+            raise
 
-    def plot_residuals(self, y_true, y_pred, title="Residual Analysis"):
+    def plot_residuals(self, y_true, y_pred, title="Residual Analysis", save_path=None):
         try:
             residuals = np.array(y_true) - np.array(y_pred)
             fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4.5))
@@ -161,9 +182,14 @@ class ModelEvaluator:
 
             plt.suptitle(title, fontsize=13, fontweight="bold")
             plt.tight_layout()
-            plt.show()
+            if save_path:
+                plt.savefig(save_path)
+                plt.close(fig)
+            else:
+                plt.show()
         except Exception as e:
             logger.error("Error in plot_residuals: %s", e)
+            raise
 
     # ==============================================================
     # 3. Deep End-to-End Diagnostics Runner
@@ -183,14 +209,33 @@ class ModelEvaluator:
             logger.info("Evaluating model predictions (task=%s, samples=%d)", self.task, len(self.y_true))
 
             if is_class:
+                from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
                 logger.info("Classification report:\n%s", classification_report(self.y_true, self.y_pred))
                 self.plot_confusion_matrix(self.y_true, self.y_pred)
 
+                acc = accuracy_score(self.y_true, self.y_pred)
+                prec = precision_score(self.y_true, self.y_pred, average='weighted', zero_division=0)
+                rec = recall_score(self.y_true, self.y_pred, average='weighted', zero_division=0)
+                f1_w = f1_score(self.y_true, self.y_pred, average='weighted', zero_division=0)
+                f1_m = f1_score(self.y_true, self.y_pred, average='macro', zero_division=0)
+                roc_auc = None
+
                 if len(np.unique(self.y_true)) == 2 and hasattr(model, "predict_proba"):
                     self.y_prob = model.predict_proba(X)[:, 1]
-                    self.plot_roc_curve(self.y_true, self.y_prob)
+                    roc_auc = self.plot_roc_curve(self.y_true, self.y_prob)
                     self.plot_precision_recall(self.y_true, self.y_prob)
 
+                self.metrics = {
+                    "task": "Classification",
+                    "accuracy": round(float(acc), 4),
+                    "precision_weighted": round(float(prec), 4),
+                    "recall_weighted": round(float(rec), 4),
+                    "f1_weighted": round(float(f1_w), 4),
+                    "f1_macro": round(float(f1_m), 4),
+                    "roc_auc": round(float(roc_auc), 4) if roc_auc is not None else None
+                }
+                return self.metrics
+
             else:
                 r2 = r2_score(self.y_true, self.y_pred)
                 rmse = np.sqrt(mean_squared_error(self.y_true, self.y_pred))
@@ -198,7 +243,7 @@ class ModelEvaluator:
                 mape = mean_absolute_percentage_error(self.y_true, self.y_pred)
 
                 n = len(self.y_true)
-                p = X.shape[1]
+                p = X.shape[1] if hasattr(X, "shape") and len(X.shape) > 1 else 1
                 adj_r2 = 1 - (1 - r2) * (n - 1) / (n - p - 1) if (n - p - 1) > 0 else r2
 
                 logger.info("Regression metrics: R2=%.4f, Adj_R2=%.4f, RMSE=%.4f, MAE=%.4f, MAPE=%.2f%%",
@@ -207,5 +252,16 @@ class ModelEvaluator:
                 self.plot_actual_vs_predicted(self.y_true, self.y_pred)
                 self.plot_residuals(self.y_true, self.y_pred)
 
+                self.metrics = {
+                    "task": "Regression",
+                    "r2": round(float(r2), 4),
+                    "adj_r2": round(float(adj_r2), 4),
+                    "rmse": round(float(rmse), 4),
+                    "mae": round(float(mae), 4),
+                    "mape": round(float(mape), 4)
+                }
+                return self.metrics
+
         except Exception as e:
             logger.error("Error in evaluate: %s", e)
+            raise

serverlex-1.2.1/serverlex/preprocessing/clean.py

@@ -0,0 +1,306 @@
+import pandas as pd
+import numpy as np
+import logging 
+
+logger=logging.getLogger("Clean")
+if not logger.handlers:
+    _h=logging.StreamHandler()
+    _h.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", "%Y-%m-%d %H:%M:%S"))
+    logger.addHandler(_h)
+    logger.setLevel(logging.INFO)
+
+class Clean:
+    def __init__(self, path_or_df):
+        if isinstance(path_or_df, pd.DataFrame):
+            self.path = None
+            self.data = path_or_df.copy()
+        else:
+            self.path = path_or_df
+            self.data = None
+
+    def load(self):
+        if not hasattr(self, 'data') or self.data is None:
+            if self.path is not None:
+                self.data = pd.read_csv(self.path)
+            else:
+                raise ValueError("No dataset or file path provided to Clean.")
+        return self.data
+
+    def separate(self):
+        self.load()
+        self.numerical_columns = self.data.select_dtypes(include='number')
+        self.categorical_columns = self.data.select_dtypes(include=['object', 'category', 'string'])
+        self.categrorical_columns = self.categorical_columns  # Backward-compatibility typo alias
+        return self.numerical_columns, self.categorical_columns
+
+    seperate = separate  # Backward-compatibility typo alias
+
+    def outliers_IQR(self, param='m', exclude=()):
+        self.separate()
+        exclude = set(exclude) if exclude else set()
+        cols = [c for c in self.numerical_columns.columns if c not in exclude]
+        if not cols:
+            return self.data
+
+        num = self.numerical_columns[cols]
+        q1 = num.quantile(0.25)
+        q3 = num.quantile(0.75)
+        iqr = q3 - q1
+        lower_bound = q1 - 1.5 * iqr
+        upper_bound = q3 + 1.5 * iqr
+
+        bad = (num < lower_bound) | (num > upper_bound)
+
+        if param in ['r', 'drop', 'remove']:
+            self.data = self.data.loc[~bad.any(axis=1)]
+            self.separate()
+            return self.data
+
+        if param in ['m', 'mean']:
+            fill = num.mean()
+        else:
+            fill = num.median()
+
+        self.data[cols] = num.mask(bad, fill, axis=1)
+        self.separate()
+        return self.data
+
+    def outliers_Z(self, param='m', threshold=None, exclude=()):
+        self.separate()
+        exclude = set(exclude) if exclude else set()
+        cols = [c for c in self.numerical_columns.columns if c not in exclude]
+        if not cols:
+            return self.data
+
+        num = self.numerical_columns[cols]
+        mean = num.mean()
+        std_dev = num.std()
+        std_dev = std_dev.replace(0, 1.0)
+        z_score = (num - mean) / std_dev
+        t = threshold if threshold is not None else 3.0
+
+        bad = z_score.abs() >= t
+
+        if param in ['r', 'drop', 'remove']:
+            self.data = self.data.loc[~bad.any(axis=1)]
+            self.separate()
+            return self.data
+
+        if param in ['m', 'mean']:
+            fill = mean
+        else:
+            fill = num.median()
+
+        self.data[cols] = num.mask(bad, fill, axis=1)
+        self.separate()
+        return self.data
+
+    def Clean_string(self, data_time=None):
+        self.separate()
+
+        # Handle date time format: data_time="col" or data_time=["col"] or data_time=["col", format]
+        if data_time is not None:
+            if isinstance(data_time, str):
+                col_name, dt_format = data_time, None
+            elif len(data_time) == 1:
+                col_name, dt_format = data_time[0], None
+            else:
+                col_name, dt_format = data_time[0], data_time[1]
+
+            if col_name in self.data.columns:
+                parsed_dates = pd.to_datetime(self.data[col_name], format='mixed', errors='coerce')
+                if dt_format:
+                    self.data[col_name] = parsed_dates.dt.strftime(dt_format)
+                else:
+                    self.data[col_name] = parsed_dates
+
+        # Standardize strings: strip leading/trailing, normalize internal whitespace to single space, lowercase
+        cat_cols = [c for c in self.categorical_columns.columns if data_time is None or c != data_time[0]]
+        for col in cat_cols:
+            orig_na = self.data[col].isna()
+            cleaned = self.data[col].astype(str).str.strip().str.lower().str.replace(r"\s+", " ", regex=True)
+            self.data[col] = cleaned.mask(orig_na, None)
+
+        self.separate()
+        return self.data
+
+    def delete_duplicates(self):
+        self.load()
+        self.data = self.data.drop_duplicates()
+        self.separate()
+        return self.data
+    def fill_missing(self, param='m'):
+        self.separate()
+        if param in ['r', 'd', 'drop']:
+            self.data = self.data.dropna()
+            self.separate()
+            return self.data
+        if param in ['m', 'mean']:
+            fill_vals = self.numerical_columns.mean()
+        else:
+            fill_vals = self.numerical_columns.median()
+        self.numerical_columns = self.numerical_columns.fillna(fill_vals)
+        self.data[self.numerical_columns.columns] = self.numerical_columns
+        self.separate()
+        return self.data
+
+    def fill_categorical_missing(self, fill_val=None):
+        self.separate()
+        for col in self.categorical_columns.columns:
+            val = fill_val if fill_val is not None else (self.data[col].mode()[0] if not self.data[col].mode().empty else "unknown")
+            self.data[col] = self.data[col].fillna(val)
+        self.separate()
+        return self.data
+
+    def clean_column_names(self):
+        self.load()
+        self.data.columns = (
+            self.data.columns.str.strip()
+            .str.lower()
+            .str.replace(r"\s+", "_", regex=True)
+            .str.replace(r"[^\w_]", "", regex=True)
+        )
+        self.separate()
+        return self.data
+
+    def drop_uninformative(self, null_threshold=0.7):
+        self.load()
+        null_pct = self.data.isnull().mean()
+        cols_to_drop = null_pct[null_pct > null_threshold].index.tolist()
+        zero_var = [col for col in self.data.columns if self.data[col].nunique(dropna=False) <= 1]
+        all_drop = list(set(cols_to_drop + zero_var))
+        if all_drop:
+            self.data = self.data.drop(columns=all_drop)
+        self.separate()
+        return self.data
+
+    def clean_numeric_strings(self, columns):
+        self.load()
+        for col in columns:
+            if col in self.data.columns:
+                cleaned = self.data[col].astype(str).str.replace(r"[\$,]", "", regex=True).str.strip()
+                self.data[col] = pd.to_numeric(cleaned, errors="coerce")
+        self.separate()
+        return self.data
+
+    def low_corr(self, target=None, threshold=0.1):
+        self.separate()
+        corr_matrix = self.numerical_columns.corr().abs()
+        if target is not None:
+            if target in corr_matrix.columns:
+                low_corr_cols = corr_matrix[target][corr_matrix[target] < threshold].index.tolist()
+                self.data = self.data.drop(columns=low_corr_cols)
+                self.separate()
+        else:
+            np.fill_diagonal(corr_matrix.values, 0)
+            low_corr_cols = corr_matrix.columns[(corr_matrix.max() < threshold)].tolist()
+            if low_corr_cols:
+                self.data = self.data.drop(columns=low_corr_cols)
+                self.separate()
+        return self.data
+
+    def summary(self):
+        self.load()
+        missing_cols = (self.data.isnull().sum() > 0).sum()
+        logger.info("Dataset summary: shape=%s, duplicates=%d, columns_with_missing=%d",
+                    self.data.shape, self.data.duplicated().sum(), missing_cols)
+        return {
+            "shape": self.data.shape,
+            "duplicates": int(self.data.duplicated().sum()),
+            "columns_with_missing": int(missing_cols)
+        }
+
+    def export(self, output_path):
+        self.load()
+        self.data.to_csv(output_path, index=False)
+        logger.info("Saved cleaned dataset to '%s'", output_path)
+
+    def total_clean(
+        self,
+        data_time=None,
+        numeric_string_cols=None,
+        missing_param='m',
+        cat_fill_val=None,
+        outlier_method='IQR',
+        outlier_param='m',
+        outlier_threshold=None,
+        exclude_outliers=None,
+        null_threshold=0.7,
+        corr_target=None,
+        corr_threshold=None,
+        export_path=None
+    ):
+        # 1. Load data
+        self.load()
+
+        # 2. Standardize column names
+        self.clean_column_names()
+
+        def _normalize_col(c):
+            import re
+            return re.sub(r"[^\w_]", "", re.sub(r"\s+", "_", str(c).strip().lower()))
+
+        if data_time is not None:
+            if isinstance(data_time, str):
+                c, fmt = data_time, None
+            elif len(data_time) == 1:
+                c, fmt = data_time[0], None
+            else:
+                c, fmt = data_time[0], data_time[1]
+
+            if c not in self.data.columns and _normalize_col(c) in self.data.columns:
+                data_time = [_normalize_col(c), fmt] if fmt else [_normalize_col(c)]
+
+        if numeric_string_cols:
+            numeric_string_cols = [
+                _normalize_col(c) if c not in self.data.columns and _normalize_col(c) in self.data.columns else c
+                for c in numeric_string_cols
+            ]
+
+        if corr_target is not None:
+            if corr_target not in self.data.columns and _normalize_col(corr_target) in self.data.columns:
+                corr_target = _normalize_col(corr_target)
+
+        # 3. Remove duplicate rows
+        self.delete_duplicates()
+
+        # 4. Drop uninformative columns (high nulls or constant values)
+        if null_threshold is not None:
+            self.drop_uninformative(null_threshold=null_threshold)
+
+        # 5. Clean currency / formatted numeric strings into actual numbers
+        if numeric_string_cols:
+            self.clean_numeric_strings(numeric_string_cols)
+
+        # 6. Clean strings & format datetime
+        self.Clean_string(data_time=data_time)
+
+        # 7. Fill missing values (numerical)
+        self.fill_missing(param=missing_param)
+
+        # 8. Fill missing values (categorical)
+        self.fill_categorical_missing(fill_val=cat_fill_val)
+
+        # 9. Handle numerical outliers with target/exclude protection
+        exclude_set = set()
+        if exclude_outliers:
+            exclude_set.update([_normalize_col(c) if _normalize_col(c) in self.data.columns else c for c in exclude_outliers])
+        if corr_target:
+            exclude_set.add(corr_target)
+
+        if outlier_method and str(outlier_method).lower() != 'none':
+            if outlier_method == 'Z':
+                self.outliers_Z(param=outlier_param, threshold=outlier_threshold, exclude=exclude_set)
+            elif outlier_method == 'IQR':
+                self.outliers_IQR(param=outlier_param, exclude=exclude_set)
+
+        # 10. Drop low correlation columns if requested
+        if corr_threshold is not None:
+            self.low_corr(target=corr_target, threshold=corr_threshold)
+
+        # 11. Export if path specified
+        if export_path:
+            self.export(export_path)
+
+        return self.data
+