serverlex 1.2.2__tar.gz → 1.2.3__tar.gz

{serverlex-1.2.2 → serverlex-1.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: serverlex
-Version: 1.2.2
+Version: 1.2.3
 Summary: Production-Safe ML Preprocessing, Diagnostics, Tuning, and Serving Platform
 Author: Serverlex Team
 License: MIT

{serverlex-1.2.2 → serverlex-1.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "serverlex"
-version = "1.2.2"
+version = "1.2.3"
 description = "Production-Safe ML Preprocessing, Diagnostics, Tuning, and Serving Platform"
 readme = "README.md"
 authors = [{ name = "Serverlex Team" }]

{serverlex-1.2.2 → serverlex-1.2.3}/serverlex/__init__.py

@@ -24,4 +24,4 @@ __all__ = [
     "ModelDeployer",
 ]
 
-__version__ = "1.2.2"
+__version__ = "1.2.3"

{serverlex-1.2.2 → serverlex-1.2.3}/serverlex/diagnostics/dataguard.py

@@ -13,7 +13,15 @@ if not logger.handlers:
 
 class DataGuard:
     def __init__(self, df=None, target=None):
-        self.data=df.copy() if df is not None else None
+        if df is not None:
+            self.data=df.copy()
+            # Automatically strip embedded header rows (e.g. from CSV concatenation)
+            min_match = max(2, len(self.data.columns) // 2)
+            header_mask = (self.data == self.data.columns).sum(axis=1) >= min_match
+            if header_mask.any():
+                self.data = self.data[~header_mask].reset_index(drop=True)
+        else:
+            self.data=None
         self.target=target
         self.warnings=[]
         self.id_cols=[]
@@ -22,8 +30,28 @@ class DataGuard:
 
     def set_data(self, df):
         self.data=df.copy()
+        if self.data is not None:
+            min_match = max(2, len(self.data.columns) // 2)
+            header_mask = (self.data == self.data.columns).sum(axis=1) >= min_match
+            if header_mask.any():
+                self.data = self.data[~header_mask].reset_index(drop=True)
         self.warnings=[]
 
+    def _get_numeric_coerced_df(self):
+        """Helper to coerce numeric-majority object columns so audit checks are not blinded by string dtypes."""
+        _UNIT_PAT = r'(?i)\s*(gb|mb|kb|tb|pb|yrs?|years?|months?|days?|hours?|hrs?|mins?|sec|seconds?|%|km|miles?|kg|lbs?)\s*$'
+        _SYM_PAT = r'[₹\$,/€£¥\s]'
+        df_coerced = self.data.copy()
+        for col in df_coerced.select_dtypes(include=['object', 'string']).columns:
+            sample = df_coerced[col].dropna().astype(str).str.strip()
+            if sample.empty or len(sample) < 5:
+                continue
+            cleaned = sample.str.replace(_UNIT_PAT, '', regex=True).str.replace(_SYM_PAT, '', regex=True).str.replace(',', '.', regex=False).str.strip()
+            coerced = pd.to_numeric(cleaned, errors='coerce')
+            if coerced.notna().mean() >= 0.70:
+                df_coerced[col] = coerced
+        return df_coerced
+
     # 1. Detect ID Columns
     def check_id_columns(self, threshold=0.95):
         try:
@@ -56,12 +84,14 @@ class DataGuard:
             if not target_to_use or target_to_use not in self.data.columns:
                 return []
 
-            num_cols=self.data.select_dtypes(include='number').copy()
+            # Use coerced numeric dataframe so string-formatted leakage (e.g. '$1,200', '0.994') is caught
+            df_coerced = self._get_numeric_coerced_df()
+            num_cols=df_coerced.select_dtypes(include='number').copy()
             if target_to_use not in num_cols.columns:
                 try:
-                    num_cols[target_to_use]=pd.to_numeric(self.data[target_to_use])
+                    num_cols[target_to_use]=pd.to_numeric(df_coerced[target_to_use], errors='coerce')
                 except Exception:
-                    num_cols[target_to_use]=self.data[target_to_use].astype('category').cat.codes
+                    num_cols[target_to_use]=df_coerced[target_to_use].astype('category').cat.codes
 
             if target_to_use in num_cols.columns:
                 corrs=num_cols.corr().abs()[target_to_use]
@@ -89,7 +119,11 @@ class DataGuard:
             if not target_to_use or target_to_use not in self.data.columns:
                 return
 
-            counts=self.data[target_to_use].value_counts(normalize=True)
+            target_series = pd.to_numeric(self.data[target_to_use], errors='coerce').dropna()
+            if target_series.empty:
+                target_series = self.data[target_to_use].dropna()
+
+            counts=target_series.value_counts(normalize=True)
             if len(counts) > 1 and len(counts) <= 20:
                 minority_prop=counts.min()
                 if minority_prop < 0.10:
@@ -107,21 +141,69 @@ class DataGuard:
     def check_cardinality(self, threshold=100):
         try:
             self.high_cardinality_cols=[]
-            cat_cols=self.data.select_dtypes(include=['object', 'category', 'string']).columns
+            # Use coerced numeric dataframe so numerical columns are not falsely treated as categoricals
+            df_coerced = self._get_numeric_coerced_df()
+            cat_cols=df_coerced.select_dtypes(include=['object', 'category', 'string']).columns
+
+            pii_cols = []
+            date_cols = []
+            generic_high_card = []
 
             for col in cat_cols:
-                n_unique=self.data[col].nunique()
-                if n_unique > threshold and col not in self.id_cols:
-                    self.high_cardinality_cols.append((col, n_unique))
+                n_unique=df_coerced[col].nunique()
+                if col in self.id_cols:
+                    continue
+
+                col_clean = re.sub(r'[^\w]', '', col.lower())
+                sample = df_coerced[col].dropna().astype(str).str.strip()
+
+                # Check if it's a datetime column
+                is_name_date = bool(re.search(r'(date|time|timestamp|dob)$', col_clean))
+                is_parse_date = (
+                    pd.to_datetime(sample.iloc[:100], format='mixed', errors='coerce').notna().mean() >= 0.70
+                    if not sample.empty else False
+                )
+                if is_name_date or is_parse_date:
+                    date_cols.append((col, n_unique))
+                    logger.info("Datetime column identified: '%s' (%d unique timestamps)", col, n_unique)
+                    continue
+
+                # Check if it's PII (name, email, phone, etc.)
+                is_pii = bool(re.search(r'(name|fullname|firstname|lastname|email|phone|address|ssn)$', col_clean))
+                if is_pii:
+                    pii_cols.append((col, n_unique))
+                    logger.warning("PII column detected: '%s' (%d unique values)", col, n_unique)
+                    continue
+
+                if n_unique > threshold:
+                    generic_high_card.append((col, n_unique))
                     logger.warning("High cardinality column '%s': %d unique values", col, n_unique)
 
-            if self.high_cardinality_cols:
+            if pii_cols:
+                self.warnings.append({
+                    "Check": "PII Detection",
+                    "Status": "WARN",
+                    "Details": f"Columns {pii_cols} appear to contain Personally Identifiable Information (PII).",
+                    "Fix": "Drop PII columns to protect user privacy and prevent overfitting."
+                })
+
+            if date_cols:
+                self.warnings.append({
+                    "Check": "Datetime Features",
+                    "Status": "INFO",
+                    "Details": f"Columns {date_cols} are datetime series rather than plain categoricals.",
+                    "Fix": "Extract datetime features (year, month, day, dayofweek) via pipeline's data_time parameter."
+                })
+
+            if generic_high_card:
                 self.warnings.append({
                     "Check": "High Cardinality",
                     "Status": "WARN",
-                    "Details": f"Categorical columns {self.high_cardinality_cols} have > {threshold} unique categories.",
+                    "Details": f"Categorical columns {generic_high_card} have > {threshold} unique categories.",
                     "Fix": "Use Target/Frequency Encoding or group rare categories."
                 })
+
+            self.high_cardinality_cols = pii_cols + generic_high_card
             return self.high_cardinality_cols
         except Exception as e:
             logger.error("Error in check_cardinality: %s", e)

{serverlex-1.2.2 → serverlex-1.2.3}/serverlex/preprocessing/clean.py

@@ -162,6 +162,19 @@ class Clean:
         self.separate()
         return self.data
 
+    def drop_embedded_headers(self):
+        """Remove rows that are copies of the header (e.g. from CSV concatenation)."""
+        self.load()
+        # A row is an embedded header if it matches ≥ half of column names exactly
+        min_match = max(2, len(self.data.columns) // 2)
+        mask = (self.data == self.data.columns).sum(axis=1) >= min_match
+        if mask.any():
+            n = int(mask.sum())
+            logger.warning("Dropping %d embedded header row(s) found in data", n)
+            self.data = self.data[~mask].reset_index(drop=True)
+        self.separate()
+        return self.data
+
     def delete_duplicates(self):
         self.load()
         self.data = self.data.drop_duplicates()
@@ -271,6 +284,9 @@ class Clean:
         # 1. Load data
         self.load()
 
+        # 1a. Strip embedded header rows (e.g. from CSV concatenation) BEFORE column rename
+        self.drop_embedded_headers()
+
         # 2. Standardize column names
         self.clean_column_names()
 

{serverlex-1.2.2 → serverlex-1.2.3}/serverlex/preprocessing/pipeline.py

@@ -1,4 +1,5 @@
 import logging
+import re
 import pandas as pd
 import numpy as np
 from serverlex.preprocessing.clean import Clean
@@ -72,6 +73,34 @@ class MLPipeline:
 
             logger.info("Fitting pipeline on %d rows (target=%s, encode=%s, scale=%s)", len(self.data), self.target, self.encode, self.scale)
 
+            # 0. Auto-detect object columns that are predominantly numeric (≥70% parseable).
+            #    This handles columns read as strings from dirty CSVs (e.g. "9.6 GB", "50 kg", "0,0")
+            #    so they are coerced to numeric rather than one-hot encoded as categoricals.
+            _UNIT_PAT = r'(?i)\s*(gb|mb|kb|tb|pb|yrs?|years?|months?|days?|hours?|hrs?|mins?|sec|seconds?|%|km|miles?|kg|lbs?)\s*$'
+            _SYM_PAT = r'[₹\$,/€£¥\s]'
+            _auto_numeric = set(numeric_string_cols or [])
+            for col in self.data.select_dtypes(include=['object', 'string']).columns:
+                if self.target and col.strip().lower() == self.target:
+                    continue
+                sample = self.data[col].dropna().astype(str).str.strip()
+                if sample.empty or len(sample) < 5:
+                    continue
+                cleaned_sample = (
+                    sample
+                    .str.replace(_UNIT_PAT, '', regex=True)
+                    .str.replace(_SYM_PAT, '', regex=True)
+                    .str.replace(',', '.', regex=False)
+                    .str.strip()
+                )
+                parse_rate = pd.to_numeric(cleaned_sample, errors='coerce').notna().mean()
+                if parse_rate >= 0.70:
+                    _auto_numeric.add(col)
+            if _auto_numeric:
+                extra = sorted(_auto_numeric - set(numeric_string_cols or []))
+                if extra:
+                    logger.info("Auto-detected numeric-majority string columns (will coerce): %s", extra)
+            numeric_string_cols = list(_auto_numeric) if _auto_numeric else numeric_string_cols
+
             # 1. Clean the training data using your Clean class with target protection
             clean_kwargs = kwargs.copy()
             if self.target:
@@ -188,12 +217,27 @@ class MLPipeline:
             out.columns=out.columns.str.strip().str.lower().str.replace(' ', '_').str.replace(r'[^\w_]', '', regex=True)
 
             # Clean numeric string columns using saved self.numeric_string_cols
+            _UNIT_PAT2 = r'(?i)\s*(gb|mb|kb|tb|pb|yrs?|years?|months?|days?|hours?|hrs?|mins?|sec|seconds?|%|km|miles?|kg|lbs?)\s*$'
+            _SYM_PAT2 = r'[₹\$,/€£¥]'
             if hasattr(self, 'numeric_string_cols') and self.numeric_string_cols:
                 for col in self.numeric_string_cols:
                     if col in out.columns:
-                        cleaned=out[col].astype(str).str.replace(r'[\$,]', '', regex=True).str.strip()
+                        cleaned=out[col].astype(str).str.replace(_SYM_PAT2, '', regex=True).str.replace(_UNIT_PAT2, '', regex=True).str.replace(',', '.', regex=False).str.strip()
                         out[col]=pd.to_numeric(cleaned, errors='coerce')
 
+            # Also coerce any num_fill column that is still string/object dtype
+            # (handles auto-detected columns stored after column name cleaning)
+            for col in list(self.num_fill.keys()):
+                if col in out.columns and (out[col].dtype == object or str(out[col].dtype).startswith('str')):
+                    cleaned = (
+                        out[col].astype(str)
+                        .str.replace(_UNIT_PAT2, '', regex=True)
+                        .str.replace(_SYM_PAT2, '', regex=True)
+                        .str.replace(',', '.', regex=False)
+                        .str.strip()
+                    )
+                    out[col] = pd.to_numeric(cleaned, errors='coerce')
+
             # 2. Extract datetime features using saved date_col
             if self.date_col and self.date_col in out.columns:
                 dt=pd.to_datetime(out[self.date_col], format='mixed', errors='coerce')

{serverlex-1.2.2 → serverlex-1.2.3}/serverlex/serving/deployer.py

@@ -2,6 +2,7 @@ import logging
 import hashlib
 import json
 import os
+import re
 import joblib
 import pandas as pd
 import numpy as np
@@ -176,8 +177,22 @@ class ModelDeployer:
         """
         clean_df = df.copy()
 
-        # 1. Snake case incoming column names
-        clean_df.columns = [c.strip().lower().replace(' ', '_').replace('$', '') for c in clean_df.columns]
+        # 1. Standardize incoming column names to match Clean.clean_column_names
+        clean_df.columns = [
+            re.sub(r"[^\w_]", "", c.strip().lower().replace(' ', '_'))
+            for c in clean_df.columns
+        ]
+
+        # 1b. Expand date_col into sub-features (mirrors pipeline.transform) so users
+        #     can pass e.g. signup_date instead of signup_date_year/month/day/dayofweek.
+        date_col = getattr(self.pipeline, "date_col", None) if self.pipeline else None
+        if date_col and date_col in clean_df.columns:
+            dt = pd.to_datetime(clean_df[date_col], format='mixed', errors='coerce')
+            clean_df[f'{date_col}_year'] = dt.dt.year
+            clean_df[f'{date_col}_month'] = dt.dt.month
+            clean_df[f'{date_col}_day'] = dt.dt.day
+            clean_df[f'{date_col}_dayofweek'] = dt.dt.dayofweek
+            clean_df = clean_df.drop(columns=[date_col])
 
         # 2. Strict mode validation checks before coercion/imputation
         if strict and self.pipeline is not None:
@@ -203,11 +218,19 @@ class ModelDeployer:
                             problems.append(f"{col}: non-numeric values")
 
             # Check missing required training features
+            date_col = getattr(self.pipeline, "date_col", None)
+            generated_date_cols = {f"{date_col}_year", f"{date_col}_month", f"{date_col}_day", f"{date_col}_dayofweek"} if date_col else set()
+
             missing_features = []
             if hasattr(self.pipeline, "num_fill"):
-                missing_features.extend([c for c in self.pipeline.num_fill.keys() if c not in clean_df.columns])
+                missing_num = [c for c in self.pipeline.num_fill.keys() if c not in clean_df.columns]
+                if date_col and date_col in clean_df.columns:
+                    missing_num = [c for c in missing_num if c not in generated_date_cols]
+                missing_features.extend(missing_num)
+
             if hasattr(self.pipeline, "cat_fill"):
                 missing_features.extend([c for c in self.pipeline.cat_fill.keys() if c not in clean_df.columns])
+
             if missing_features:
                 problems.append(f"missing required training features: {missing_features}")
 
@@ -215,9 +238,15 @@ class ModelDeployer:
                 raise ValueError("Strict deployment error: " + "; ".join(problems))
 
         # 3. Handle numerical missing columns and currency parsing
+        # Note: date_col sub-features were already expanded above; the source date col is gone.
+        generated_date_cols = {f"{date_col}_year", f"{date_col}_month", f"{date_col}_day", f"{date_col}_dayofweek"} if date_col else set()
+
         if hasattr(self.pipeline, "num_fill") and self.pipeline.num_fill:
             for num_col, fill_val in self.pipeline.num_fill.items():
                 if num_col not in clean_df.columns:
+                    if num_col in generated_date_cols:
+                        # Already expanded from date_col above; skip imputation
+                        continue
                     clean_df[num_col] = fill_val
                     logger.info("Missing numerical feature '%s' imputed with training baseline: %s", num_col, fill_val)
                 else:
@@ -226,6 +255,7 @@ class ModelDeployer:
                         stripped = col_vals.astype(str).str.replace(r'[\$,]', '', regex=True).str.strip()
                         clean_df[num_col] = pd.to_numeric(stripped, errors='coerce').fillna(fill_val)
 
+
         # 4. Handle categorical missing columns
         if hasattr(self.pipeline, "cat_fill") and self.pipeline.cat_fill:
             for cat_col, fill_val in self.pipeline.cat_fill.items():
@@ -247,7 +277,7 @@ class ModelDeployer:
 
         return self._validate_and_sanitize(df, strict=strict)
 
-    def predict(self, raw_input, strict=False):
+    def predict(self, raw_input, strict=False, threshold=None):
         if self.pipeline is None or self.model is None:
             raise AttributeError("Pipeline and model must be loaded before calling predict().")
 
@@ -259,6 +289,12 @@ class ModelDeployer:
         else:
             X = transformed_res
 
+        # If a custom decision threshold is provided and model supports probabilities
+        if threshold is not None and hasattr(self.model, "predict_proba"):
+            proba = self.model.predict_proba(X)
+            if proba.ndim == 2 and proba.shape[1] >= 2:
+                return (proba[:, 1] >= threshold).astype(int)
+
         predictions = self.model.predict(X)
         return predictions
 

{serverlex-1.2.2 → serverlex-1.2.3}/serverlex.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: serverlex
-Version: 1.2.2
+Version: 1.2.3
 Summary: Production-Safe ML Preprocessing, Diagnostics, Tuning, and Serving Platform
 Author: Serverlex Team
 License: MIT

{serverlex-1.2.2 → serverlex-1.2.3}/setup.py

@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="serverlex",
-    version="1.2.2",
+    version="1.2.3",
     author="Serverlex Team",
     description="Production-Safe ML Preprocessing, Diagnostics, Tuning, and Serving Platform",
     long_description=long_description,

{serverlex-1.2.2 → serverlex-1.2.3}/tests/test_audit_remediation.py

@@ -249,3 +249,52 @@ def test_deployer_hmac_missing_key_clear_error(tmp_path):
     with pytest.raises(ValueError, match="saved with HMAC authenticity protection"):
         ModelDeployer.load(bundle_path)
 
+
+def test_clean_drops_embedded_header_rows():
+    """1.2.3 Regression: Embedded duplicate header rows from CSV concatenation are dropped."""
+    df = pd.DataFrame({
+        "age": ["25", "30", "age", "45"],
+        "fee": ["100", "200", "fee", "300"],
+        "plan": ["basic", "standard", "plan", "premium"]
+    })
+    cleaner = Clean(df)
+    cleaner.drop_embedded_headers()
+    assert len(cleaner.data) == 3
+    assert "age" not in cleaner.data["age"].values
+
+
+def test_pipeline_auto_coerces_numeric_strings_and_serving(tmp_path):
+    """1.2.3 Regression: Columns with units/symbols (9.6 GB, 0,0) are auto-coerced to numeric and deploy cleanly."""
+    df = pd.DataFrame({
+        "signup_date": ["2024-01-01", "2024-02-01", "2024-03-01", "2024-04-01"] * 10,
+        "usage_gb": ["6.5", "9.6 GB", "0,0", "12.2"] * 10,
+        "fee_per_gb": ["45.5", "50.0", "10.2", "88.1"] * 10,
+        "auto_renew": ["0", "1", "0", "1"] * 10,
+        "churn": [0, 1, 0, 1] * 10
+    })
+    pipeline = MLPipeline()
+    pipeline.fit(df, target="churn", data_time=["signup_date"], encode="onehot", scale="standard")
+
+    # usage_gb and fee_per_gb should be numeric, not one-hot encoded
+    assert "usage_gb" in pipeline.num_fill
+    assert "fee_per_gb" in pipeline.num_fill
+    assert "usage_gb" not in pipeline.cat_categories
+
+    X, y = pipeline.transform(df)
+    assert X.shape[1] < 15  # not blown up into dozens of categories
+
+    model = RandomForestClassifier(n_estimators=10, random_state=42).fit(X, y)
+    deployer = ModelDeployer(pipeline=pipeline, model=model)
+    bundle_file = str(tmp_path / "auto_bundle.joblib")
+    deployer.save(bundle_file)
+
+    loaded = ModelDeployer.load(bundle_file)
+    sample = {
+        "signup_date": "2024-05-01",
+        "usage_gb": 10.5,
+        "fee_per_gb": 47.92,
+        "auto_renew": 1
+    }
+    pred = loaded.predict(sample, strict=True)
+    assert len(pred) == 1
+

{serverlex-1.2.2 → serverlex-1.2.3}/tests/test_deployer.py

@@ -97,3 +97,17 @@ def test_deployer_batch_prediction(trained_bundle):
     ]
     preds = deployer.predict(batch)
     assert len(preds) == 2
+
+
+def test_deployer_custom_decision_threshold(trained_bundle):
+    deployer = ModelDeployer.load(trained_bundle)
+    sample = {"age": 30.0, "income": 50000.0, "city": "New York"}
+    proba = deployer.predict_proba(sample)[0][1]
+
+    # Extreme low threshold forces 1 (positive)
+    pred_low = deployer.predict(sample, threshold=0.0)
+    assert pred_low[0] == 1
+
+    # Extreme high threshold forces 0 (negative)
+    pred_high = deployer.predict(sample, threshold=1.0)
+    assert pred_high[0] == 0