serverlex 1.0.0__tar.gz

serverlex-1.0.0/Clean.py

@@ -0,0 +1,261 @@
+import pandas as pd
+import numpy as np
+import logging 
+
+logger=logging.getLogger("Clean")
+if not logger.handlers:
+    _h=logging.StreamHandler()
+    _h.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", "%Y-%m-%d %H:%M:%S"))
+    logger.addHandler(_h)
+    logger.setLevel(logging.INFO)
+
+class Clean:
+    def __init__(self,path):
+        self.path=path
+        self.data=None
+    def load(self):
+        try:
+            if not hasattr(self, 'data') or self.data is None:
+                self.data=pd.read_csv(self.path)
+        except  Exception as e:
+            print(e)
+    def seperate(self):
+        try:
+            self.load()
+            self.numerical_columns=self.data.select_dtypes(include='number')
+            self.categrorical_columns=self.data.select_dtypes(include=['object','category'])
+        except Exception as e:
+            print(e)
+    def outliers_IQR(self,param):
+        self.seperate()
+        Q1 = self.numerical_columns.quantile(0.25)
+        Q3 = self.numerical_columns.quantile(0.75)
+        IQR = Q3 - Q1
+        lower_bound = Q1 - 1.5 * IQR
+        upper_bound = Q3 + 1.5 * IQR
+        condition=((self.numerical_columns >=lower_bound) & (self.numerical_columns<=upper_bound)).all(axis=1)
+        if param=='r':
+            self.data = self.data.loc[condition]
+            self.numerical_columns = self.numerical_columns[condition]
+            return
+        if param=='m':
+            column_mean=self.numerical_columns.mean()
+            self.numerical_columns=self.numerical_columns.mask(~condition,column_mean,axis=1)
+            self.data[self.numerical_columns.columns] = self.numerical_columns
+            return
+        column_median=self.numerical_columns.median()
+        self.numerical_columns=self.numerical_columns.mask(~condition,column_median,axis=1)
+        self.data[self.numerical_columns.columns]=self.numerical_columns
+    def outliers_Z(self,param,threshold=None):
+        self.seperate()
+        mean=self.numerical_columns.mean()
+        x_diff_mean=(self.numerical_columns)-mean
+        std_dev=self.numerical_columns.std()
+        z_score=x_diff_mean/std_dev
+        t = threshold if threshold is not None else 3
+        is_valid = z_score.abs() < t
+        if param == 'r':
+            condition = is_valid.all(axis=1)
+            self.data = self.data.loc[condition]
+            self.numerical_columns = self.numerical_columns.loc[condition]
+            return
+        elif param == 'm':
+            self.numerical_columns = self.numerical_columns.mask(~is_valid, mean, axis=1)
+            self.data[self.numerical_columns.columns] = self.numerical_columns
+            return
+        column_median=self.numerical_columns.median()
+        self.numerical_columns=self.numerical_columns.mask(~is_valid,column_median,axis=1)
+        self.data[self.numerical_columns.columns]=self.numerical_columns
+    def Clean_string(self, data_time=None):
+        try:
+            self.seperate()
+
+            # Handle date time format from user: data_time=["column name of date time", format you want it to be] or else None
+            if data_time is not None:
+                col_name = data_time[0]
+                dt_format = data_time[1]
+                if col_name in self.data.columns:
+                    parsed_dates = pd.to_datetime(self.data[col_name], format='mixed', errors='coerce')
+                    self.data[col_name] = parsed_dates.dt.strftime(dt_format)
+
+            # Eliminate gaps between categories and lower them all for categorical columns
+            cat_cols = [c for c in self.categrorical_columns.columns if data_time is None or c != data_time[0]]
+            for col in cat_cols:
+                self.data[col] = self.data[col].astype(str).str.lower().str.replace(r"\s+", "", regex=True)
+                self.data[col] = self.data[col].mask(self.categrorical_columns[col].isna(), None)
+            self.categrorical_columns = self.data[self.categrorical_columns.columns]
+        except Exception as e:
+            print(e)
+    def delete_duplicates(self):
+        try:
+            self.load()
+            self.data = self.data.drop_duplicates()
+            self.seperate()
+        except Exception as e:
+            print(e)
+    def fill_missing(self, param='m'):
+        try:
+            self.seperate()
+            if param in ['r', 'd', 'drop']:
+                self.data = self.data.dropna()
+                self.seperate()
+                return
+            if param in ['m', 'mean']:
+                fill_vals = self.numerical_columns.mean()
+            else:
+                fill_vals = self.numerical_columns.median()
+            self.numerical_columns = self.numerical_columns.fillna(fill_vals)
+            self.data[self.numerical_columns.columns] = self.numerical_columns
+        except Exception as e:
+            print(e)
+    def fill_categorical_missing(self, fill_val=None):
+        try:
+            self.seperate()
+            for col in self.categrorical_columns.columns:
+                val = fill_val if fill_val is not None else (self.data[col].mode()[0] if not self.data[col].mode().empty else "unknown")
+                self.data[col] = self.data[col].fillna(val)
+            self.categrorical_columns = self.data[self.categrorical_columns.columns]
+        except Exception as e:
+            print(e)
+    def clean_column_names(self):
+        try:
+            self.load()
+            self.data.columns = (
+                self.data.columns.str.strip()
+                .str.lower()
+                .str.replace(r"\s+", "_", regex=True)
+                .str.replace(r"[^\w_]", "", regex=True)
+            )
+            self.seperate()
+        except Exception as e:
+            print(e)
+    def drop_uninformative(self, null_threshold=0.7):
+        try:
+            self.load()
+            null_pct = self.data.isnull().mean()
+            cols_to_drop = null_pct[null_pct > null_threshold].index.tolist()
+            zero_var = [col for col in self.data.columns if self.data[col].nunique(dropna=False) <= 1]
+            self.data = self.data.drop(columns=list(set(cols_to_drop + zero_var)))
+            self.seperate()
+        except Exception as e:
+            print(e)
+    def clean_numeric_strings(self, columns):
+        try:
+            self.load()
+            for col in columns:
+                if col in self.data.columns:
+                    cleaned = self.data[col].astype(str).str.replace(r"[\$,]", "", regex=True)
+                    self.data[col] = pd.to_numeric(cleaned, errors="coerce")
+            self.seperate()
+        except Exception as e:
+            print(e)
+    def low_corr(self, target=None, threshold=0.1):
+        try:
+            self.seperate()
+            corr_matrix = self.numerical_columns.corr().abs()
+            if target is not None:
+                if target in corr_matrix.columns:
+                    low_corr_cols = corr_matrix[target][corr_matrix[target] < threshold].index.tolist()
+                    self.data = self.data.drop(columns=low_corr_cols)
+                    self.seperate()
+            else:
+                np.fill_diagonal(corr_matrix.values, 0)
+                low_corr_cols = corr_matrix.columns[(corr_matrix.max() < threshold)].tolist()
+                self.data = self.data.drop(columns=low_corr_cols)
+                self.seperate()
+        except Exception as e:
+            print(e)
+    def summary(self):
+        try:
+            self.load()
+            missing_cols = (self.data.isnull().sum() > 0).sum()
+            logger.info("Dataset summary: shape=%s, duplicates=%d, columns_with_missing=%d",
+                        self.data.shape, self.data.duplicated().sum(), missing_cols)
+        except Exception as e:
+            logger.error("Error in summary: %s", e)
+    def export(self, output_path):
+        try:
+            self.data.to_csv(output_path, index=False)
+            logger.info("Saved cleaned dataset to '%s'", output_path)
+        except Exception as e:
+            logger.error("Error in export: %s", e)
+    def total_clean(
+        self,
+        data_time=None,
+        numeric_string_cols=None,
+        missing_param='m',
+        cat_fill_val=None,
+        outlier_method='IQR',
+        outlier_param='m',
+        outlier_threshold=None,
+        null_threshold=0.7,
+        corr_target=None,
+        corr_threshold=None,
+        export_path=None
+    ):
+        try:
+            # 1. Load data
+            self.load()
+
+            # 2. Standardize column names
+            self.clean_column_names()
+
+            # Normalize user-passed column names to match cleaned column names if needed
+            def _normalize_col(c):
+                import re
+                return re.sub(r"[^\w_]", "", re.sub(r"\s+", "_", str(c).strip().lower()))
+
+            if data_time is not None:
+                c, fmt = data_time[0], data_time[1]
+                if c not in self.data.columns and _normalize_col(c) in self.data.columns:
+                    data_time = [_normalize_col(c), fmt]
+
+            if numeric_string_cols:
+                numeric_string_cols = [
+                    _normalize_col(c) if c not in self.data.columns and _normalize_col(c) in self.data.columns else c
+                    for c in numeric_string_cols
+                ]
+
+            if corr_target is not None:
+                if corr_target not in self.data.columns and _normalize_col(corr_target) in self.data.columns:
+                    corr_target = _normalize_col(corr_target)
+
+            # 3. Remove duplicate rows
+            self.delete_duplicates()
+
+            # 4. Drop uninformative columns (high nulls or constant values)
+            if null_threshold is not None:
+                self.drop_uninformative(null_threshold=null_threshold)
+
+            # 5. Clean currency / formatted numeric strings into actual numbers
+            if numeric_string_cols:
+                self.clean_numeric_strings(numeric_string_cols)
+
+            # 6. Clean strings & format datetime
+            self.Clean_string(data_time=data_time)
+
+            # 7. Fill missing values (numerical)
+            self.fill_missing(param=missing_param)
+
+            # 8. Fill missing values (categorical)
+            self.fill_categorical_missing(fill_val=cat_fill_val)
+
+            # 9. Handle numerical outliers
+            if outlier_method == 'Z':
+                self.outliers_Z(param=outlier_param, threshold=outlier_threshold)
+            elif outlier_method == 'IQR':
+                self.outliers_IQR(param=outlier_param)
+
+            # 10. Drop low correlation columns if requested
+            if corr_threshold is not None:
+                self.low_corr(target=corr_target, threshold=corr_threshold)
+
+            # 11. Export if path specified
+            if export_path:
+                self.export(export_path)
+
+            return self.data
+        except Exception as e:
+            print(e)
+            return self.data
+

serverlex-1.0.0/DataCheck.py

@@ -0,0 +1,165 @@
+import logging
+import pandas as pd
+import numpy as np
+import re
+
+logger=logging.getLogger("DataGuard")
+if not logger.handlers:
+    _h=logging.StreamHandler()
+    _h.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", "%Y-%m-%d %H:%M:%S"))
+    logger.addHandler(_h)
+    logger.setLevel(logging.INFO)
+
+
+class DataGuard:
+    def __init__(self, df=None):
+        self.data=df.copy() if df is not None else None
+        self.warnings=[]
+        self.id_cols=[]
+        self.leakage_cols=[]
+        self.high_cardinality_cols=[]
+
+    def set_data(self, df):
+        self.data=df.copy()
+        self.warnings=[]
+
+    # 1. Detect ID Columns
+    def check_id_columns(self):
+        try:
+            self.id_cols=[]
+            for col in self.data.columns:
+                unique_ratio=self.data[col].nunique(dropna=True) / len(self.data)
+                is_name_id=bool(re.search(r'(id|uuid|key|ssn|index|identifier)$', col.lower()))
+
+                if (unique_ratio > 0.95 and is_name_id) or (unique_ratio == 1.0 and self.data[col].dtype == 'object'):
+                    self.id_cols.append(col)
+                    logger.warning("Potential unique ID column detected: '%s' (uniqueness: %.1f%%)", col, unique_ratio * 100)
+
+            if self.id_cols:
+                self.warnings.append({
+                    "Check": "ID Columns",
+                    "Status": "WARN",
+                    "Details": f"Columns {self.id_cols} appear to be unique IDs.",
+                    "Fix": "Drop ID columns to prevent overfitting."
+                })
+            return self.id_cols
+        except Exception as e:
+            logger.error("Error in check_id_columns: %s", e)
+
+    # 2. Check Target Leakage
+    def check_target_leakage(self, target, threshold=0.95):
+        try:
+            self.leakage_cols=[]
+            if target not in self.data.columns:
+                return
+
+            num_cols=self.data.select_dtypes(include='number')
+            if target in num_cols.columns:
+                corrs=num_cols.corr().abs()[target]
+                for col, val in corrs.items():
+                    if col != target and val >= threshold:
+                        self.leakage_cols.append((col, round(val, 4)))
+                        logger.error("Suspicious target leakage: feature '%s' has correlation %.4f with target '%s'", col, val, target)
+
+            if self.leakage_cols:
+                self.warnings.append({
+                    "Check": "Target Leakage",
+                    "Status": "FAIL",
+                    "Details": f"Features {self.leakage_cols} have correlation >= {threshold} with target '{target}'.",
+                    "Fix": "Verify if these columns contain future data."
+                })
+            return self.leakage_cols
+        except Exception as e:
+            logger.error("Error in check_target_leakage: %s", e)
+
+    # 3. Check Class Imbalance
+    def check_class_imbalance(self, target):
+        try:
+            if target not in self.data.columns:
+                return
+
+            counts=self.data[target].value_counts(normalize=True)
+            if len(counts) <= 20: # Classification
+                max_prop=counts.max()
+                min_prop=counts.min()
+                imbalance_ratio=round(max_prop / min_prop, 2)
+
+                if max_prop >= 0.70:
+                    status="WARN" if max_prop < 0.90 else "FAIL"
+                    logger.warning("Class imbalance in '%s': dominant class is %.1f%% (ratio %.2f:1)", target, max_prop * 100, imbalance_ratio)
+                    self.warnings.append({
+                        "Check": "Class Imbalance",
+                        "Status": status,
+                        "Details": f"Dominant class represents {max_prop:.1%} (ratio: {imbalance_ratio}:1).",
+                        "Fix": "Use class_weight='balanced' or resampling."
+                    })
+        except Exception as e:
+            logger.error("Error in check_class_imbalance: %s", e)
+
+    # 4. Check High-Cardinality Categoricals
+    def check_cardinality(self, threshold=30):
+        try:
+            self.high_cardinality_cols=[]
+            cat_cols=self.data.select_dtypes(include=['object', 'category']).columns
+
+            for col in cat_cols:
+                n_unique=self.data[col].nunique()
+                if n_unique > threshold:
+                    self.high_cardinality_cols.append((col, n_unique))
+                    logger.warning("High cardinality in category '%s': %d unique values", col, n_unique)
+
+            if self.high_cardinality_cols:
+                self.warnings.append({
+                    "Check": "High Cardinality",
+                    "Status": "WARN",
+                    "Details": f"Columns {self.high_cardinality_cols} exceed {threshold} unique values.",
+                    "Fix": "Use label encoding or group rare categories."
+                })
+        except Exception as e:
+            logger.error("Error in check_cardinality: %s", e)
+
+    # 5. Check Infinite Values
+    def check_infinite_values(self):
+        try:
+            num_cols=self.data.select_dtypes(include='number').columns
+            inf_cols=[]
+            for col in num_cols:
+                if np.isinf(self.data[col]).any():
+                    inf_cols.append(col)
+                    logger.error("Infinite values detected in column '%s'", col)
+
+            if inf_cols:
+                self.warnings.append({
+                    "Check": "Infinite Values",
+                    "Status": "FAIL",
+                    "Details": f"Columns {inf_cols} contain inf / -inf.",
+                    "Fix": "Replace inf values with NaN or bounds."
+                })
+        except Exception as e:
+            logger.error("Error in check_infinite_values: %s", e)
+
+    # ==========================
+    # Master Health Audit Runner
+    # ==========================
+    def run_audit(self, target=None):
+        try:
+            self.warnings=[]
+            logger.info("Running data quality audit on %d rows, %d columns", len(self.data), len(self.data.columns))
+
+            self.check_id_columns()
+            self.check_infinite_values()
+            self.check_cardinality()
+
+            if target:
+                self.check_target_leakage(target)
+                self.check_class_imbalance(target)
+
+            if not self.warnings:
+                logger.info("Quality audit completed: all checks passed")
+                return pd.DataFrame([{"Check": "All Checks", "Status": "PASS", "Details": "No issues found", "Fix": "None"}])
+
+            report_df=pd.DataFrame(self.warnings)
+            logger.info("Quality audit completed: %d warnings/failures flagged", len(self.warnings))
+            return report_df
+        except Exception as e:
+            logger.error("Error in run_audit: %s", e)

serverlex-1.0.0/DriftDetector.py

@@ -0,0 +1,90 @@
+import logging
+import pandas as pd
+import numpy as np
+from scipy.stats import ks_2samp, chi2_contingency
+
+logger=logging.getLogger("DriftDetector")
+if not logger.handlers:
+    _h=logging.StreamHandler()
+    _h.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", "%Y-%m-%d %H:%M:%S"))
+    logger.addHandler(_h)
+    logger.setLevel(logging.INFO)
+
+
+class DriftDetector:
+    def __init__(self, reference_data, alpha=0.05):
+        """
+        reference_data: Historical baseline (training data)
+        alpha: Significance threshold for statistical hypothesis tests (default: 0.05)
+        """
+        self.ref_data=reference_data.copy() if isinstance(reference_data, pd.DataFrame) else pd.DataFrame(reference_data)
+        self.alpha=alpha
+        self.drift_results=[]
+
+    def detect_drift(self, current_data):
+        """
+        Compares incoming production data against reference data:
+        - Numerical features: Two-Sample Kolmogorov-Smirnov test (ks_2samp)
+        - Categorical features: Chi-Square Contingency test
+        """
+        try:
+            cur_data=current_data.copy() if isinstance(current_data, pd.DataFrame) else pd.DataFrame(current_data)
+            self.drift_results=[]
+            drift_count=0
+
+            common_cols=[c for c in self.ref_data.columns if c in cur_data.columns]
+            logger.info("Running drift detection across %d features (alpha=%.3f)", len(common_cols), self.alpha)
+
+            for col in common_cols:
+                # 1. Numerical Drift via Kolmogorov-Smirnov test
+                if pd.api.types.is_numeric_dtype(self.ref_data[col]):
+                    ref_vals=self.ref_data[col].dropna()
+                    cur_vals=cur_data[col].dropna()
+
+                    if len(ref_vals) > 0 and len(cur_vals) > 0:
+                        stat, p_val = ks_2samp(ref_vals, cur_vals)
+                        is_drift = p_val < self.alpha
+                        if is_drift:
+                            drift_count += 1
+                            logger.warning("Data drift detected in '%s' (KS=%.4f, p-value=%.5f)", col, stat, p_val)
+
+                        self.drift_results.append({
+                            "Feature": col,
+                            "Type": "Numerical",
+                            "Statistic": round(stat, 4),
+                            "P_Value": round(p_val, 5),
+                            "Drift_Detected": is_drift
+                        })
+
+                # 2. Categorical Drift via Chi-Square test
+                else:
+                    ref_counts=self.ref_data[col].astype(str).value_counts()
+                    cur_counts=cur_data[col].astype(str).value_counts()
+
+                    # Align categories
+                    all_cats=list(set(ref_counts.index).union(set(cur_counts.index)))
+                    ref_aligned=[ref_counts.get(cat, 0) for cat in all_cats]
+                    cur_aligned=[cur_counts.get(cat, 0) for cat in all_cats]
+
+                    contingency_table=np.array([ref_aligned, cur_aligned])
+                    if contingency_table.sum() > 0:
+                        stat, p_val, _, _ = chi2_contingency(contingency_table)
+                        is_drift = p_val < self.alpha
+                        if is_drift:
+                            drift_count += 1
+                            logger.warning("Categorical shift detected in '%s' (Chi2=%.4f, p-value=%.5f)", col, stat, p_val)
+
+                        self.drift_results.append({
+                            "Feature": col,
+                            "Type": "Categorical",
+                            "Statistic": round(stat, 4),
+                            "P_Value": round(p_val, 5),
+                            "Drift_Detected": is_drift
+                        })
+
+            report_df=pd.DataFrame(self.drift_results)
+            logger.info("Drift analysis complete: %d/%d features drifted", drift_count, len(common_cols))
+            return report_df
+        except Exception as e:
+            logger.error("Error in detect_drift: %s", e)
+            return pd.DataFrame()

serverlex-1.0.0/EnsembleBuilder.py

@@ -0,0 +1,78 @@
+import logging
+import pandas as pd
+import numpy as np
+
+from sklearn.ensemble import VotingClassifier, VotingRegressor, StackingClassifier, StackingRegressor
+from sklearn.linear_model import LogisticRegression, Ridge
+from sklearn.model_selection import cross_val_score, StratifiedKFold, KFold
+
+logger=logging.getLogger("EnsembleBuilder")
+if not logger.handlers:
+    _h=logging.StreamHandler()
+    _h.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", "%Y-%m-%d %H:%M:%S"))
+    logger.addHandler(_h)
+    logger.setLevel(logging.INFO)
+
+
+class EnsembleBuilder:
+    """
+    Automated Model Ensembling (Voting & Stacking):
+    Combines diverse algorithms to improve predictive performance and reduce variance.
+    """
+
+    def __init__(self, task="classification"):
+        self.task=task.lower()
+
+    def build_voting_ensemble(self, estimators, voting="soft"):
+        """
+        Builds a Voting Ensemble:
+        Averages predictions across multiple models (soft = probability-weighted, hard = majority vote).
+        estimators: list of (name, model) tuples, e.g. [('rf', rf), ('gb', gb), ('lr', lr)]
+        """
+        try:
+            logger.info("Building Voting Ensemble (%d estimators, mode=%s)", len(estimators), voting)
+            if self.task == "classification":
+                ensemble = VotingClassifier(estimators=estimators, voting=voting)
+            else:
+                ensemble = VotingRegressor(estimators=estimators)
+            return ensemble
+        except Exception as e:
+            logger.error("Error in build_voting_ensemble: %s", e)
+
+    def build_stacking_ensemble(self, estimators, final_estimator=None, cv=5):
+        """
+        Builds a Stacking Ensemble:
+        Uses a meta-learner to learn how to best combine predictions from base models.
+        """
+        try:
+            logger.info("Building Stacking Ensemble (%d base estimators, cv=%d)", len(estimators), cv)
+            if self.task == "classification":
+                meta = final_estimator or LogisticRegression(max_iter=1000)
+                ensemble = StackingClassifier(estimators=estimators, final_estimator=meta, cv=cv, n_jobs=-1)
+            else:
+                meta = final_estimator or Ridge()
+                ensemble = StackingRegressor(estimators=estimators, final_estimator=meta, cv=cv, n_jobs=-1)
+            return ensemble
+        except Exception as e:
+            logger.error("Error in build_stacking_ensemble: %s", e)
+
+    def evaluate_ensemble(self, ensemble, X, y, cv=5):
+        """Evaluates an ensemble model using cross-validation."""
+        try:
+            if not isinstance(y, pd.Series):
+                y = pd.Series(y)
+
+            if self.task == "classification":
+                kf = StratifiedKFold(n_splits=cv, shuffle=True, random_state=42)
+                acc = cross_val_score(ensemble, X, y, cv=kf, scoring="accuracy")
+                f1 = cross_val_score(ensemble, X, y, cv=kf, scoring="f1_weighted")
+                logger.info("Ensemble CV Accuracy: %.4f (+/- %.4f) | F1: %.4f", acc.mean(), acc.std(), f1.mean())
+                return {"Accuracy": round(acc.mean(), 4), "Std_Accuracy": round(acc.std(), 4), "F1": round(f1.mean(), 4)}
+            else:
+                kf = KFold(n_splits=cv, shuffle=True, random_state=42)
+                r2 = cross_val_score(ensemble, X, y, cv=kf, scoring="r2")
+                rmse = cross_val_score(ensemble, X, y, cv=kf, scoring="neg_root_mean_squared_error")
+                logger.info("Ensemble CV R2: %.4f (+/- %.4f) | RMSE: %.4f", r2.mean(), r2.std(), -rmse.mean())
+                return {"R2": round(r2.mean(), 4), "Std_R2": round(r2.std(), 4), "RMSE": round(-rmse.mean(), 4)}
+        except Exception as e:
+            logger.error("Error in evaluate_ensemble: %s", e)