osmotic-data-filter 0.1.8__py3-none-any.whl

osmotic_data_filter/__init__.py

@@ -0,0 +1,228 @@
+"""
+Osmotic Data Filter (ODF) - v0.1.8 Bulletproof Edition
+======================================================
+A physical-chemistry inspired, safe, and hybrid outlier handling framework for Python.
+Combines Interquartile Range (IQR) for critical boundaries with osmotic distance
+adjustments for smooth value scaling without involuntary data deletion.
+
+Author: Jana Elkasrawy
+License: MIT
+"""
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+def detect_outliers_preview(df, columns=None, iqr_factor=1.5):
+    if df is None or df.empty:
+        return pd.DataFrame()
+    if columns is None:
+        columns = [col for col in df.columns if pd.api.types.is_numeric_dtype(df[col])]
+    report = []
+    for col in columns:
+        if col not in df.columns or not pd.api.types.is_numeric_dtype(df[col]):
+            continue
+        series = df[col].replace([np.inf, -np.inf], np.nan).dropna()
+        if series.empty:
+            continue
+        q1 = series.quantile(0.25)
+        q3 = series.quantile(0.75)
+        iqr = q3 - q1
+        if pd.isna(iqr) or iqr <= 0:
+            iqr = 0.0
+        lower_limit = q1 - (iqr_factor * iqr)
+        upper_limit = q3 + (iqr_factor * iqr)
+        outliers_count = ((series < lower_limit) | (series > upper_limit)).sum()
+        total_len = len(series)
+        outlier_pct = (outliers_count / total_len * 100) if total_len > 0 else 0.0
+        report.append({
+            'Column': col,
+            'Q1': round(float(q1), 4),
+            'Median': round(float(series.median()), 4),
+            'Q3': round(float(q3), 4),
+            'IQR': round(float(iqr), 4),
+            'Lower_Critical_Limit': round(float(lower_limit), 4),
+            'Upper_Critical_Limit': round(float(upper_limit), 4),
+            'Outliers_Count': int(outliers_count),
+            'Outlier_Percentage (%)': round(float(outlier_pct), 2)
+        })
+    return pd.DataFrame(report)
+
+def partition_uncertain_cases(df, target_column, iqr_factor=1.5):
+    if df is None or df.empty or target_column not in df.columns:
+        return pd.DataFrame(), pd.DataFrame()
+    series = df[target_column].replace([np.inf, -np.inf], np.nan).dropna()
+    if series.empty:
+        return df.copy(), pd.DataFrame()
+    q1 = series.quantile(0.25)
+    q3 = series.quantile(0.75)
+    iqr = q3 - q1
+    if pd.isna(iqr) or iqr <= 0:
+        iqr = 0.0
+    lower_limit = q1 - (iqr_factor * iqr)
+    upper_limit = q3 + (iqr_factor * iqr)
+    temp_series = df[target_column].replace([np.inf, -np.inf], np.nan)
+    outlier_mask = (temp_series < lower_limit) | (temp_series > upper_limit)
+    outlier_mask = outlier_mask.fillna(False)
+    return df[~outlier_mask].copy(), df[outlier_mask].copy()
+
+def Osmotic_Data_FLTR(df, columns=None, iqr_factor=1.5, osmotic_scale=0.5, action='adjust', round_to_integer=False):
+    if df is None:
+        return pd.DataFrame()
+    if action not in ['adjust', 'flag']:
+        raise ValueError("Action must be either 'adjust' or 'flag'")
+
+    if osmotic_scale not in ['adj_auto', 'adj_dnmc']:
+        if not isinstance(osmotic_scale, (int, float)) or pd.isna(osmotic_scale):
+            raise ValueError("Osmotic scale must be 'adj_auto', 'adj_dnmc', or a numeric value.")
+        if not (0.0 < osmotic_scale < 1.0):
+            raise ValueError("Numeric osmotic scale must be strictly greater than 0 and less than 1.")
+
+    df_result = df.copy()
+    if columns is None:
+        columns = [col for col in df_result.columns if pd.api.types.is_numeric_dtype(df_result[col])]
+
+    for col in columns:
+        if col not in df_result.columns or not pd.api.types.is_numeric_dtype(df_result[col]):
+            continue
+        series = df_result[col].replace([np.inf, -np.inf], np.nan)
+        series_clean = series.dropna()
+        if series_clean.empty:
+            continue
+
+        q1 = series_clean.quantile(0.25)
+        q3 = series_clean.quantile(0.75)
+        iqr = q3 - q1
+        if pd.isna(iqr) or iqr <= 0:
+            iqr = 0.0
+        lower_limit = q1 - (iqr_factor * iqr)
+        upper_limit = q3 + (iqr_factor * iqr)
+
+        if action == 'flag':
+            flag_col = f'{col}_is_outlier'
+            if flag_col in df_result.columns:
+                raise ValueError(f"Column collision: '{flag_col}' already exists!")
+            is_out = (series < lower_limit) | (series > upper_limit)
+            df_result[flag_col] = is_out.fillna(False)
+
+        elif action == 'adjust':
+            upper_mask = (series > upper_limit) & (~series.isna())
+            lower_mask = (series < lower_limit) & (~series.isna())
+            outlier_mask = upper_mask | lower_mask
+            natural_mask = (~outlier_mask) & (~series.isna())
+
+            if outlier_mask.any() or df_result[col].dtype != float:
+                df_result[col] = df_result[col].astype(float)
+                series = df_result[col]
+
+            natural_values = series[natural_mask].to_numpy()
+
+            if osmotic_scale == 'adj_auto':
+                if len(natural_values) >= 2:
+                    n1 = natural_values[:-1]
+                    n2 = natural_values[1:]
+                    denoms = n1 + n2
+                    ratios = np.where(denoms == 0, 0.5, np.abs((n1 - n2) / denoms))
+                    global_ratio = np.mean(ratios)
+                else:
+                    global_ratio = 0.5
+
+                if upper_mask.any():
+                    excess = series[upper_mask] - upper_limit
+                    df_result.loc[upper_mask, col] = np.abs(global_ratio * excess) + upper_limit
+                if lower_mask.any():
+                    deficit = lower_limit - series[lower_mask]
+                    df_result.loc[lower_mask, col] = lower_limit - np.abs(global_ratio * deficit)
+
+            elif osmotic_scale == 'adj_dnmc':
+                if len(natural_values) < 2:
+                    active_ratio = 0.5
+                    if upper_mask.any():
+                        excess = series[upper_mask] - upper_limit
+                        df_result.loc[upper_mask, col] = np.abs(active_ratio * excess) + upper_limit
+                    if lower_mask.any():
+                        deficit = lower_limit - series[lower_mask]
+                        df_result.loc[lower_mask, col] = lower_limit - np.abs(active_ratio * deficit)
+                else:
+                    natural_indices = np.where(natural_mask.to_numpy())[0]
+                    full_series = series.to_numpy()
+
+                    for idx in np.where(outlier_mask.to_numpy())[0]:
+                        val = full_series[idx]
+                        prev_nat_indices = natural_indices[natural_indices < idx]
+                        if len(prev_nat_indices) >= 2:
+                            n1 = full_series[prev_nat_indices[-1]]
+                            n2 = full_series[prev_nat_indices[-2]]
+                            denom = n1 + n2
+                            local_ratio = np.abs((n1 - n2) / denom) if denom != 0 else 0.5
+                        else:
+                            local_ratio = 0.5
+
+                        if val > upper_limit:
+                            excess = val - upper_limit
+                            df_result.iloc[idx, df_result.columns.get_loc(col)] = np.abs(local_ratio * excess) + upper_limit
+                        elif val < lower_limit:
+                            deficit = lower_limit - val
+                            df_result.iloc[idx, df_result.columns.get_loc(col)] = lower_limit - np.abs(local_ratio * deficit)
+
+            else:
+                scale_val = float(osmotic_scale)
+                if upper_mask.any():
+                    excess = series[upper_mask] - upper_limit
+                    df_result.loc[upper_mask, col] = upper_limit + (excess * scale_val)
+                if lower_mask.any():
+                    deficit = lower_limit - series[lower_mask]
+                    df_result.loc[lower_mask, col] = lower_limit - (deficit * scale_val)
+
+            df_result[col] = df_result[col].astype(float)
+            if round_to_integer:
+                df_result[col] = np.round(df_result[col]).astype(float)
+
+    return df_result
+
+def check_skewness_health(df_before, df_after, columns=None):
+    if df_before is None or df_after is None:
+        return pd.DataFrame()
+    if columns is None:
+        columns = [col for col in df_before.columns if pd.api.types.is_numeric_dtype(df_before[col])]
+    health_report = []
+    for col in columns:
+        if col in df_before.columns and col in df_after.columns:
+            if not pd.api.types.is_numeric_dtype(df_before[col]) or not pd.api.types.is_numeric_dtype(df_after[col]):
+                continue
+            s_before = df_before[col].replace([np.inf, -np.inf], np.nan).dropna()
+            s_after = df_after[col].replace([np.inf, -np.inf], np.nan).dropna()
+            if s_before.empty or s_after.empty:
+                continue
+            health_report.append({
+                'Column': col,
+                'Skew_Before': round(float(s_before.skew()), 4) if len(s_before) > 2 else np.nan,
+                'Skew_After': round(float(s_after.skew()), 4) if len(s_after) > 2 else np.nan,
+                'Kurtosis_Before': round(float(s_before.kurtosis()), 4) if len(s_before) > 3 else np.nan,
+                'Kurtosis_After': round(float(s_after.kurtosis()), 4) if len(s_after) > 3 else np.nan,
+                'Mean_Shift': round(float(s_after.mean() - s_before.mean()), 4),
+                'Std_Change': round(float(s_after.std() - s_before.std()), 4)
+            })
+    return pd.DataFrame(health_report)
+
+def plot_osmotic_dashboard(df_before, df_after, column):
+    fig, axes = plt.subplots(2, 2, figsize=(14, 8))
+    fig.suptitle(f"Osmotic Data Filter Analysis: '{column}'", fontsize=14, fontweight='bold')
+    axes[0, 0].set_title("Original Distribution (With Outliers)")
+    axes[1, 0].set_title("Original Boxplot")
+    axes[0, 1].set_title("Osmotic Adjusted Distribution")
+    axes[1, 1].set_title("Osmotic Adjusted Boxplot")
+
+    if df_before is not None and df_after is not None and column in df_before.columns and column in df_after.columns:
+        s_before = df_before[column].replace([np.inf, -np.inf], np.nan).dropna()
+        s_after = df_after[column].replace([np.inf, -np.inf], np.nan).dropna()
+        if not s_before.empty:
+            sns.histplot(s_before, kde=True, ax=axes[0, 0], color='crimson')
+            sns.boxplot(x=s_before, ax=axes[1, 0], color='crimson')
+        if not s_after.empty:
+            sns.histplot(s_after, kde=True, ax=axes[0, 1], color='teal')
+            sns.boxplot(x=s_after, ax=axes[1, 1], color='teal')
+
+    plt.tight_layout()
+    return fig

osmotic_data_filter-0.1.8.dist-info/METADATA

@@ -0,0 +1,131 @@
+Metadata-Version: 2.4
+Name: osmotic-data-filter
+Version: 0.1.8
+Summary: A physical chemistry-inspired, safe, and hybrid data bias handling framework
+Author-email: Jana Elkasrawy <kasrawymahmoud@gmail.com>
+License-Expression: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+
+
+readme_content = r"""# Osmotic Data Filter (ODF)
+
+
+**Osmotic Data Filter (ODF)** is a scientifically grounded, non-destructive data preprocessing Python library designed to handle outliers and anomalies without row deletion or hard clamping. Engineered to meet strict scientific reproducibility standards (e.g., ISEF requirements), ODF compresses extreme statistical deviations toward safe critical boundaries while fully preserving dataset volume, structure, and integrity.
+
+---
+
+## 🌟 Key Features (v0.1.8)
+
+* **Non-Destructive Outlier Adjustment (`action='adjust'`)**: Compresses outliers toward lower/upper limit boundaries without dropping rows or clamping values to artificial caps.
+* **Smart Scaling Modes (`osmotic_scale`)**: Adaptive scaling designed for both independent and time-series/contextual datasets.
+* **Flexible Post-Adjustment Rounding (`round_to_integer`)**: Optional rounding of adjusted values to nearest integers while strictly preserving `float` data types to prevent DataFrame schema conflicts.
+* **Outlier Flagging (`action='flag'`)**: Adds explicit boolean indicator columns (`<column>_is_outlier`) without altering underlying numerical values.
+* **Comprehensive Visual & Health Metrics**: Built-in functions (`check_skewness_health` and `plot_osmotic_dashboard`) for quantitative skewness reporting and automated dual-panel exploratory dashboards.
+
+---
+
+## 🧭 Choosing the Right `osmotic_scale` (Guide)
+
+ODF provides three strict modes for the `osmotic_scale` parameter, tailored to different data types:
+
+1. **`'adj_dnmc'` (Local Dynamic - *Default*):**
+   * **When to use:** Ideal for time-series data, sequential measurements, or datasets with local clusters and high local variance.
+   * **Why:** It adapts to the local neighborhood of each outlier, preserving structural trends and contextual continuity without destroying local variations.
+
+2. **`'adj_auto'` (Global Automatic):**
+   * **When to use:** Best for cross-sectional data, independent statistical measurements, or uniform distributions.
+   * **Why:** It evaluates global distribution ratios and applies a consistent overall compression logic.
+
+3. **Fixed Numeric Scale (`float` between 0 and 1, e.g., `0.3`):**
+   * **When to use:** When you need strict, deterministic, or experimentally controlled compression ratios mandated by specific research protocols.
+
+---
+
+
+## 🚀 Quick Start Example
+
+```python
+import numpy as np
+import pandas as pd
+import osmotic_data_filter as odf
+
+# 1. Create sample dataset with extreme outliers
+data = {
+    'feature_col': [10, 12, 11, 14, 15, 12, 900, -400, 13]
+}
+df = pd.DataFrame(data)
+
+# 2. Apply Osmotic Data Filtering with Local Dynamic Scale (Default) & Integer Rounding
+df_processed = odf.Osmotic_Data_FLTR(
+    df,
+    columns=['feature_col'],
+    osmotic_scale='adj_dnmc',
+    action='adjust',
+    round_to_integer=True
+)
+
+print(df_processed)
+print("Data Type:", df_processed['feature_col'].dtype)
+
+# 3. Check Skewness & Statistical Health
+health_report = odf.check_skewness_health(
+    df, df_processed, columns=['feature_col']
+)
+print(health_report)
+
+# 4. Render Comparison Dashboard
+fig = odf.plot_osmotic_dashboard(
+    df, df_processed, column='feature_col'
+)
+```
+
+## 📖 API Reference
+
+### `Osmotic_Data_FLTR(df, columns, osmotic_scale='adj_dnmc', action='adjust', round_to_integer=False)`
+
+Main execution function for dataset normalization and adjustment.
+
+- **`df`**: Input pandas DataFrame containing numerical features (Required).
+- **`columns`**: List of target column names to inspect and filter (Required).
+- **`osmotic_scale`**: Mode of scaling. Accepts `'adj_auto'`, `'adj_dnmc'`, or float numeric (`0 < scale < 1`). Default is `'adj_dnmc'`.
+- **`action`**: Operation mode. Use `'adjust'` for non-destructive compression or `'flag'` for tagging. Default is `'adjust'`.
+- **`round_to_integer`**: Boolean. If `True`, rounds adjusted values to nearest integers while maintaining `float` dtype. Default is `False`.
+
+### `check_skewness_health(df_original, df_filtered, columns)`
+
+Computes initial vs. adjusted skewness, IQR variance, and distribution delta metrics.
+
+### `plot_osmotic_dashboard(df_original, df_filtered, column)`
+
+Generates a publication-ready Matplotlib/Seaborn visualization comparing Original vs. Osmotically Adjusted KDE distributions and Boxplots.
+
+## 🔬 Scientific Methodology & Reproducibility
+
+1. **Row Count Conservation**: Unlike standard truncation methods, ODF guarantees `N_initial == N_final`, preventing sample bias in downstream analytical or machine learning workflows.
+2. **Schema & Type Consistency**: Enforces continuous mathematical representation (`float64`) regardless of integer rounding flags, ensuring full compatibility with Scikit-Learn, PyTorch, and TensorFlow pipelines.
+3. **Bounded Adjustments**: Strict boundary validation enforces `0 < scale < 1`, preventing divergence or inverse scaling artifacts.
+
+---
+
+## 📜 License
+
+Distributed under the **MIT License**. See `LICENSE` file for more information.
+"""
+
+# Create README.md
+with open("README.md", "w", encoding="utf-8") as file:
+    file.write(readme_content)
+
+print("README.md created successfully!")
+
+# Download the file
+from google.colab import files
+files.download("README.md")

osmotic_data_filter-0.1.8.dist-info/RECORD

@@ -0,0 +1,5 @@
+osmotic_data_filter/__init__.py,sha256=lRa8NYH6k3i0BALAcfXQrI58AHytD2vMk33Xg-Lux_s,10646
+osmotic_data_filter-0.1.8.dist-info/METADATA,sha256=Qm0Ro34CrJe4hqdN4_Q9BGDI77C2e6aRgY-P2wsNJJI,5768
+osmotic_data_filter-0.1.8.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+osmotic_data_filter-0.1.8.dist-info/top_level.txt,sha256=fLhYRpz2QLLSHGQoZj6aECa8yio7WbJzn14sa2VSKEY,20
+osmotic_data_filter-0.1.8.dist-info/RECORD,,

osmotic_data_filter-0.1.8.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

osmotic_data_filter-0.1.8.dist-info/top_level.txt

@@ -0,0 +1 @@
+osmotic_data_filter