sdscaler 0.2.0__py3-none-any.whl

sdscaler/__init__.py

@@ -0,0 +1,3 @@
+from .scaler import SDScaler
+
+__version__ = "0.2.0"

sdscaler/helpers.py

@@ -0,0 +1,46 @@
+# figures out which scaling method fits a column best
+# just checks for outliers and skew, nothing fancy
+
+import numpy as np
+
+
+def skewness(x):
+    x = np.asarray(x, dtype=float)
+    if len(x) < 3:
+        return 0.0
+
+    mean = np.nanmean(x)
+    std = np.nanstd(x)
+    if std == 0:
+        return 0.0
+
+    return np.nanmean(((x - mean) / std) ** 3)
+
+
+def outlier_frac(x):
+    # classic 1.5*IQR rule from stats class
+    x = np.asarray(x, dtype=float)
+    q1, q3 = np.nanpercentile(x, [25, 75])
+    iqr = q3 - q1
+
+    if iqr == 0:
+        return 0.0
+
+    low = q1 - 1.5 * iqr
+    high = q3 + 1.5 * iqr
+    outliers = (x < low) | (x > high)
+    return np.nansum(outliers) / len(x)
+
+
+def pick_method(x, skew_cutoff=1.0, outlier_cutoff=0.05):
+    if outlier_frac(x) > outlier_cutoff:
+        return "robust"
+
+    if abs(skewness(x)) > skew_cutoff:
+        # log only makes sense if the column is strictly positive, otherwise
+        # fall back to minmax like before
+        if np.nanmin(x) > 0:
+            return "log"
+        return "minmax"
+
+    return "standard"

sdscaler/scaler.py

@@ -0,0 +1,262 @@
+import warnings
+
+import numpy as np
+import pandas as pd
+
+from .helpers import pick_method
+
+METHODS = ("auto", "standard", "minmax", "robust", "log")
+
+
+class SDScaler:
+    """
+    basically StandardScaler/MinMaxScaler/RobustScaler but it picks one
+    for you per column instead of you having to guess which one to use.
+
+    scaler = SDScaler()
+    scaled = scaler.fit_transform(df)
+    scaler.summary()   # shows what got picked for each column
+
+    set method to "standard"/"minmax"/"robust"/"log" if you want to force
+    one instead of letting it decide (default is "auto")
+
+    pass log_cols if you want specific columns log-scaled regardless of
+    what auto-pick would've chosen - useful for stuff like income that's
+    always heavily skewed. auto-pick will also choose log on its own for
+    skewed, strictly-positive columns, log_cols just forces it.
+
+    if X is a dataframe, non-numeric columns get left alone (passthrough)
+    instead of crashing - only numeric columns actually get scaled.
+    """
+
+    def __init__(self, method="auto", outlier_cutoff=0.05, skew_cutoff=1.0, log_cols=None):
+        if method not in METHODS:
+            raise ValueError(f"method has to be one of {METHODS}, got '{method}'")
+
+        self.method = method
+        self.outlier_cutoff = outlier_cutoff
+        self.skew_cutoff = skew_cutoff
+        self.log_cols = log_cols or []
+
+        self.stats_ = {}
+        self.columns_ = None
+        self.numeric_columns_ = None
+        self.is_df = False
+        self.fitted = False
+
+    def fit(self, X):
+        self.is_df = isinstance(X, pd.DataFrame)
+
+        if self.is_df:
+            self.columns_ = list(X.columns)
+            self.numeric_columns_ = [
+                c for c in self.columns_ if pd.api.types.is_numeric_dtype(X[c])
+            ]
+            nan_cols = [c for c in self.numeric_columns_ if X[c].isna().any()]
+            if nan_cols:
+                warnings.warn(
+                    f"these columns have missing values, stats were computed "
+                    f"ignoring them but the scaled output will still have NaN "
+                    f"in those spots: {nan_cols}"
+                )
+            data_lookup = {c: X[c].to_numpy(dtype=float) for c in self.numeric_columns_}
+        else:
+            X = np.asarray(X, dtype=float)
+            if X.ndim == 1:
+                X = X.reshape(-1, 1)
+            self.columns_ = list(range(X.shape[1]))
+            self.numeric_columns_ = list(self.columns_)
+            data_lookup = {c: X[:, i] for i, c in enumerate(self.columns_)}
+
+        self.stats_ = {}
+        for col in self.numeric_columns_:
+            col_data = data_lookup[col]
+            if col in self.log_cols:
+                method = "log"
+            elif self.method == "auto":
+                method = pick_method(col_data, self.skew_cutoff, self.outlier_cutoff)
+            else:
+                method = self.method
+            self.stats_[col] = self._fit_one(col_data, method)
+
+        self.fitted = True
+        return self
+
+    def transform(self, X):
+        if not self.fitted:
+            raise RuntimeError("call fit() before transform()")
+
+        if self.is_df:
+            out = X.copy()
+            for col in self.numeric_columns_:
+                out[col] = self._apply_one(X[col].to_numpy(dtype=float), self.stats_[col])
+            return out
+
+        X = np.asarray(X, dtype=float)
+        squeeze = X.ndim == 1
+        if squeeze:
+            X = X.reshape(-1, 1)
+
+        out = np.empty_like(X, dtype=float)
+        for i, col in enumerate(self.numeric_columns_):
+            out[:, i] = self._apply_one(X[:, i], self.stats_[col])
+
+        return out.ravel() if squeeze else out
+
+    def fit_transform(self, X):
+        return self.fit(X).transform(X)
+
+    def inverse_transform(self, X):
+        if not self.fitted:
+            raise RuntimeError("call fit() before inverse_transform()")
+
+        if self.is_df:
+            out = X.copy()
+            for col in self.numeric_columns_:
+                out[col] = self._undo_one(X[col].to_numpy(dtype=float), self.stats_[col])
+            return out
+
+        X = np.asarray(X, dtype=float)
+        squeeze = X.ndim == 1
+        if squeeze:
+            X = X.reshape(-1, 1)
+
+        out = np.empty_like(X, dtype=float)
+        for i, col in enumerate(self.numeric_columns_):
+            out[:, i] = self._undo_one(X[:, i], self.stats_[col])
+
+        return out.ravel() if squeeze else out
+
+    def summary(self):
+        if not self.fitted:
+            raise RuntimeError("call fit() before summary()")
+
+        rows = []
+        for col in self.columns_:
+            if col in self.stats_:
+                rows.append({"column": col, "method": self.stats_[col]["method"]})
+            else:
+                rows.append({"column": col, "method": "passthrough (not numeric)"})
+        return pd.DataFrame(rows)
+
+    def to_column_transformer(self):
+        """
+        builds an sklearn ColumnTransformer that mirrors whatever this
+        scaler decided, in case you'd rather plug it into an sklearn
+        Pipeline instead of calling .transform() directly. needs sklearn.
+        non-numeric columns get passed through untouched, same as here.
+        """
+        if not self.fitted:
+            raise RuntimeError("call fit() before to_column_transformer()")
+
+        try:
+            from sklearn.compose import ColumnTransformer
+            from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
+        except ImportError:
+            raise ImportError("to_column_transformer() needs scikit-learn installed")
+
+        transformers = []
+        for col, stat in self.stats_.items():
+            if stat["method"] == "standard":
+                transformers.append((f"{col}_standard", StandardScaler(), [col]))
+            elif stat["method"] == "minmax":
+                transformers.append((f"{col}_minmax", MinMaxScaler(), [col]))
+            elif stat["method"] == "robust":
+                transformers.append((f"{col}_robust", RobustScaler(), [col]))
+            elif stat["method"] == "log":
+                transformers.append((f"{col}_log", _LogScaler(), [col]))
+
+        return ColumnTransformer(transformers, remainder="passthrough")
+
+    # internal, don't call directly
+
+    def _fit_one(self, x, method):
+        if method == "standard":
+            mean, std = np.nanmean(x), np.nanstd(x)
+            return {"method": "standard", "mean": mean, "std": std or 1.0}
+
+        if method == "minmax":
+            lo, hi = np.nanmin(x), np.nanmax(x)
+            span = hi - lo
+            return {"method": "minmax", "min": lo, "max": hi, "span": span or 1.0}
+
+        if method == "robust":
+            med = np.nanmedian(x)
+            q1, q3 = np.nanpercentile(x, [25, 75])
+            iqr = q3 - q1
+            return {"method": "robust", "median": med, "iqr": iqr or 1.0}
+
+        if method == "log":
+            m = np.nanmin(x)
+            # shift so everything's positive before taking the log - if the
+            # column's already all positive this is just offset=0 and does
+            # nothing
+            offset = (1 - m) if m <= 0 else 0.0
+            shifted = np.clip(x + offset, 1e-6, None)
+            logged = np.log(shifted)
+            mean, std = np.nanmean(logged), np.nanstd(logged)
+            return {"method": "log", "offset": offset, "mean": mean, "std": std or 1.0}
+
+        raise ValueError(f"unknown method {method}")
+
+    def _apply_one(self, x, s):
+        x = np.asarray(x, dtype=float)
+        if s["method"] == "standard":
+            return (x - s["mean"]) / s["std"]
+        if s["method"] == "minmax":
+            return (x - s["min"]) / s["span"]
+        if s["method"] == "robust":
+            return (x - s["median"]) / s["iqr"]
+        if s["method"] == "log":
+            shifted = np.clip(x + s["offset"], 1e-6, None)
+            logged = np.log(shifted)
+            return (logged - s["mean"]) / s["std"]
+
+    def _undo_one(self, x, s):
+        x = np.asarray(x, dtype=float)
+        if s["method"] == "standard":
+            return x * s["std"] + s["mean"]
+        if s["method"] == "minmax":
+            return x * s["span"] + s["min"]
+        if s["method"] == "robust":
+            return x * s["iqr"] + s["median"]
+        if s["method"] == "log":
+            logged = x * s["std"] + s["mean"]
+            shifted = np.exp(logged)
+            return shifted - s["offset"]
+
+    def __repr__(self):
+        return f"SDScaler(method='{self.method}')"
+
+
+class _LogScaler:
+    """
+    small helper class used by to_column_transformer() for log-method
+    columns - sklearn's ColumnTransformer needs something with its own
+    fit/transform, this is just our log logic wrapped up that way.
+    """
+
+    def fit(self, X, y=None):
+        x = np.asarray(X, dtype=float).ravel()
+        m = np.nanmin(x)
+        self.offset_ = (1 - m) if m <= 0 else 0.0
+        shifted = np.clip(x + self.offset_, 1e-6, None)
+        logged = np.log(shifted)
+        self.mean_ = np.nanmean(logged)
+        self.std_ = np.nanstd(logged) or 1.0
+        return self
+
+    def transform(self, X):
+        x = np.asarray(X, dtype=float).ravel()
+        shifted = np.clip(x + self.offset_, 1e-6, None)
+        logged = np.log(shifted)
+        return ((logged - self.mean_) / self.std_).reshape(-1, 1)
+
+    def fit_transform(self, X, y=None):
+        return self.fit(X, y).transform(X)
+
+    def get_params(self, deep=True):
+        return {}
+
+    def set_params(self, **params):
+        return self

sdscaler-0.2.0.dist-info/METADATA

@@ -0,0 +1,155 @@
+Metadata-Version: 2.4
+Name: sdscaler
+Version: 0.2.0
+Summary: auto-scaler that picks standard/minmax/robust scaling per column
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.20
+Requires-Dist: pandas>=1.2
+Dynamic: license-file
+
+# SDScaler
+
+Small auto-scaler for tabular data. Instead of manually picking StandardScaler
+vs MinMaxScaler vs RobustScaler for each column, this just looks at the column
+and picks one for you (checks for outliers and skew). You can also just force
+one method if you don't trust the auto part.
+
+Made this for a uni project, kept it simple on purpose - no sklearn
+dependency for the core logic (sklearn's only needed if you use
+`to_column_transformer()`), just numpy and pandas.
+
+## Install
+
+\```
+cd SDScaler
+pip install -e .
+\```
+
+## Usage
+
+\```python
+import pandas as pd
+from sdscaler import SDScaler
+
+df = pd.DataFrame({
+    "age": [22, 25, 130, 28, 24],
+    "income": [30000, 32000, 31000, 500000, 29000],
+})
+
+scaler = SDScaler()
+scaled = scaler.fit_transform(df)
+
+print(scaled)
+print(scaler.summary())   # which method got used per column
+\```
+
+Force one method instead of auto:
+
+\```python
+scaler = SDScaler(method="standard")   # or "minmax" / "robust" / "log"
+\```
+
+Undo the scaling later:
+
+\```python
+scaler.inverse_transform(scaled)
+\```
+
+Remember to fit only on your training data, then just `.transform()` the
+test set with those same stats - don't fit_transform both separately or
+you'll leak info / get mismatched scales.
+
+\```python
+train_scaled = scaler.fit_transform(train_df)
+test_scaled = scaler.transform(test_df)
+\```
+
+## How the auto part decides
+
+For each numeric column:
+- if more than 5% of points are outliers (1.5*IQR rule) -> robust
+- else if it's pretty skewed (skew > 1) and strictly positive -> log
+- else if it's pretty skewed but has zero/negative values -> minmax
+- otherwise -> standard
+
+Both cutoffs can be changed:
+
+\```python
+SDScaler(outlier_cutoff=0.1, skew_cutoff=1.5)
+\```
+
+Heads up - on small datasets one weird value can be enough to trip the
+outlier check even if the column mostly looks normal, so don't be surprised
+if a small demo dataset gets more "robust" columns than you'd expect.
+
+## Forcing log scaling on specific columns
+
+\```python
+scaler = SDScaler(log_cols=["income"])
+\```
+
+This scales `income` with log regardless of what auto-pick would've chosen,
+everything else still gets decided normally. Log scaling shifts the column
+to be positive first if it isn't already, so it won't blow up on zeros or
+negative values, but it's really meant for stuff that's naturally always
+positive and skewed (income, population, prices, that kind of thing).
+
+## Non-numeric columns
+
+If you pass a dataframe that still has a categorical/text column in it (like
+forgot to encode it, or it's a column you're not scaling on purpose), SDScaler
+just leaves it alone instead of erroring out. Only numeric columns get touched.
+
+\```python
+scaler.summary()
+#   column   method
+# 0    age   standard
+# 1   city   passthrough (not numeric)
+\```
+
+## Missing values
+
+If a column has NaN in it, `.fit()` still works and warns you about which
+columns have missing data. The stats (mean, median, etc.) get computed
+ignoring the NaNs, but the scaled output will still have NaN in those exact
+spots - didn't want to silently fill anything in without you knowing.
+
+## Using it with sklearn Pipelines
+
+\```python
+scaler = SDScaler().fit(df)
+ct = scaler.to_column_transformer()
+\```
+
+Builds an sklearn `ColumnTransformer` using the same decisions SDScaler
+already made, for anyone who'd rather have it inside a Pipeline instead of
+calling `.transform()` directly. Non-numeric columns get passed through here
+too. Needs scikit-learn installed, only used if you actually call this.
+
+## Files
+
+\```
+sdscaler/
+  scaler.py    - the SDScaler class
+  helpers.py   - skew/outlier detection logic
+examples/
+  example_usage.py
+tests/
+  test_scaler.py
+\```
+
+## Running tests
+
+\```
+pip install pytest
+pytest tests/
+\```
+
+## Todo / ideas
+
+- frequency encoding style option for very high-cardinality numeric IDs
+- maybe a plot function to compare before/after distributions
+- log scaling on negative-heavy data currently just shifts everything, could
+  be smarter about it for columns that are mostly negative

sdscaler-0.2.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+sdscaler/__init__.py,sha256=iciwye9z1GmmNgOwHUROXqzCi92TyHUolqHLRbmuhDs,52
+sdscaler/helpers.py,sha256=zNvuP4VWiHMqTVNthlVv6D6DiTOM1f4sa99NpWMAb60,1064
+sdscaler/scaler.py,sha256=MsTasiPXLZXIJ87Lly1KA6iuwjOe1Rp8GlsNN5-ByiM,9511
+sdscaler-0.2.0.dist-info/licenses/LICENSE,sha256=ESYyLizI0WWtxMeS7rGVcX3ivMezm-HOd5WdeOh-9oU,1056
+sdscaler-0.2.0.dist-info/METADATA,sha256=zuODTHU0ct4lHGIAhNSd8rZ6tCSHIu952hMOXb-9IOA,4460
+sdscaler-0.2.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+sdscaler-0.2.0.dist-info/top_level.txt,sha256=nb1MK05loALSkqBz1_ZMHqxHQcEK9goD6xXA-xw8Fh4,9
+sdscaler-0.2.0.dist-info/RECORD,,

sdscaler-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sdscaler-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sdscaler-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sdscaler