sd-auto-encoder 0.3.0__py3-none-any.whl

sd_auto_encoder-0.3.0.dist-info/METADATA

@@ -0,0 +1,120 @@
+Metadata-Version: 2.4
+Name: sd-auto-encoder
+Version: 0.3.0
+Summary: auto-picks onehot/ordinal/binary encoding per categorical column
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=1.2
+Dynamic: license-file
+
+# SDEncoder
+
+Auto-picks an encoding method per categorical column instead of you manually
+deciding onehot vs ordinal vs label encoding for each one. Companion to
+SDScaler, same idea but for categorical data instead of numeric.
+
+Two things in here:
+- `SDEncoder` - for feature columns (X)
+- `SDTargetEncoder` - for your target column (y)
+
+They're kept separate on purpose. Label-style integer codes are fine for a
+target (a model just needs distinct class ids, order doesn't matter), but
+doing that on a *feature* column tricks a lot of models into thinking there's
+a numeric relationship between categories that don't actually have one.
+`SDEncoder` will refuse to touch a Series for this reason, and `SDTargetEncoder`
+will refuse a DataFrame.
+
+## Install
+
+```
+cd SDEncoder
+pip install -e .
+```
+
+## Usage
+
+```python
+import pandas as pd
+from sdencoder import SDEncoder, SDTargetEncoder
+
+df = pd.DataFrame({
+    "color": ["red", "blue", "green", "blue"],
+    "size": ["S", "M", "L", "M"],
+    "is_member": ["yes", "no", "yes", "yes"],
+})
+
+encoder = SDEncoder(ordinal_maps={"size": ["S", "M", "L"]})
+X_encoded = encoder.fit_transform(df)
+print(X_encoded)
+print(encoder.summary())
+```
+
+For the target:
+
+```python
+y = pd.Series(["approved", "denied", "approved"])
+target_encoder = SDTargetEncoder()
+y_encoded = target_encoder.fit_transform(y)
+```
+
+Same train/test rule as SDScaler - fit on train, just transform on test:
+
+```python
+X_train_enc = encoder.fit_transform(X_train)
+X_test_enc = encoder.transform(X_test)
+```
+
+## How SDEncoder decides
+
+For each object/category column:
+- if you gave it an `ordinal_maps` entry for that column -> uses your order
+- 2 categories -> binary (0/1)
+- more than 2 but under `onehot_cutoff` (default 10) -> onehot
+- more than `onehot_cutoff` -> falls back to ordinal, since onehot on
+  something like a "city" column with 200 values would blow up your
+  feature count for not much benefit
+
+Numeric columns are left completely alone.
+
+`ordinal_maps` is the important one to use correctly - only pass a real
+order there for columns that actually have one (like S/M/L, or
+low/medium/high). Don't use it just to avoid getting a onehot column.
+
+## Unseen categories
+
+If `.transform()` sees a category that wasn't there during `.fit()`, it
+becomes `NaN` for ordinal/binary columns, or an all-zero row for onehot
+columns. That's intentional - better to notice missing data than have it
+silently mapped to some made-up number.
+
+## ColumnTransformer
+
+If you'd rather use this inside an sklearn `Pipeline`:
+
+```python
+encoder = SDEncoder(ordinal_maps={"size": ["S", "M", "L"]}).fit(df)
+ct = encoder.to_column_transformer()
+```
+
+This just builds an sklearn `ColumnTransformer` using the same decisions
+SDEncoder already made. Needs scikit-learn installed, only used if you
+actually call this method.
+
+## Files
+
+```
+sdencoder/
+  encoder.py   - SDEncoder, for X
+  target.py    - SDTargetEncoder, for y
+examples/
+  example_usage.py
+tests/
+  test_encoder.py
+```
+
+## Todo / ideas
+
+- frequency encoding as another fallback option for high-cardinality columns
+- inverse_transform for onehot columns (currently raises NotImplementedError)
+- handle missing values (NaN) as their own category instead of erroring later

sd_auto_encoder-0.3.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+sd_auto_encoder-0.3.0.dist-info/licenses/LICENSE,sha256=ESYyLizI0WWtxMeS7rGVcX3ivMezm-HOd5WdeOh-9oU,1056
+sdencoder/__init__.py,sha256=k1MoH1x2uyLHrWxU-HDWGhxqJHOUGjGLg2--puD1Qes,90
+sdencoder/encoder.py,sha256=Uon-kmLrrhdz3RkRyGtimx1JBB6VAJESwayiDLEmADc,9533
+sdencoder/target.py,sha256=7VbtfkHSm_8eWOgCh8_O6UyHPT88yxHFaVp0EZ8a22A,2392
+sd_auto_encoder-0.3.0.dist-info/METADATA,sha256=9UIDaC50N-lKJfvMV7ECTEvFW5iMewgrURfX8anMYGA,3616
+sd_auto_encoder-0.3.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+sd_auto_encoder-0.3.0.dist-info/top_level.txt,sha256=FjkiQcQPbRudyfba0RAq7EgeDUnVJc4b1LJFiYfVsM0,10
+sd_auto_encoder-0.3.0.dist-info/RECORD,,

sd_auto_encoder-0.3.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sd_auto_encoder-0.3.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sd_auto_encoder-0.3.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sdencoder

sdencoder/__init__.py

@@ -0,0 +1,4 @@
+from .encoder import SDEncoder
+from .target import SDTargetEncoder
+
+__version__ = "0.3.0"

sdencoder/encoder.py

@@ -0,0 +1,230 @@
+import pandas as pd
+
+METHODS = ("onehot", "ordinal", "binary", "target")
+
+
+class SDEncoder:
+    """
+    encodes categorical feature columns - picks onehot, ordinal, or binary
+    per column depending on how many categories it has.
+
+    encoder = SDEncoder()
+    X_encoded = encoder.fit_transform(X)
+    encoder.summary()   # see what got picked per column
+
+    pass ordinal_maps for columns that actually have a real order, like
+    ordinal_maps={"education": ["high school", "bachelors", "masters", "phd"]}
+    anything not listed there just gets auto-picked based on category count.
+
+    don't use this on your target (y), use SDTargetEncoder for that instead.
+    label-style integer codes are fine for a target since the model just
+    needs distinct class ids, but doing that on a feature column makes the
+    model think there's a numeric order between categories that don't
+    actually have one (unless you gave it an ordinal_map on purpose).
+
+    pass target_encode_cols if you want specific high-cardinality columns
+    replaced with the (smoothed) mean of y per category instead of falling
+    back to ordinal. this one's opt-in only, never auto-picked, since it
+    needs y and carries more overfitting risk than the other three. if you
+    use it, fit() needs y: encoder.fit(X, y)
+
+    note on the target method: this does simple mean + smoothing, not the
+    full cross-fold fitting sklearn's own TargetEncoder does internally, so
+    it's a bit more overfit-prone on the training set itself. fine for a
+    class project, but for anything more serious use sklearn's TargetEncoder
+    directly with fit_transform() on train.
+    """
+
+    def __init__(self, onehot_cutoff=10, ordinal_maps=None, drop_first=False,
+                 target_encode_cols=None, target_smoothing=10):
+        self.onehot_cutoff = onehot_cutoff
+        self.ordinal_maps = ordinal_maps or {}
+        self.drop_first = drop_first
+        self.target_encode_cols = target_encode_cols or []
+        self.target_smoothing = target_smoothing
+
+        self.stats_ = {}
+        self.columns_ = None
+        self.cat_columns_ = None
+        self.fitted = False
+
+    def fit(self, X, y=None):
+        if isinstance(X, pd.Series):
+            raise TypeError(
+                "SDEncoder wants a DataFrame of features, not a Series. "
+                "if this is your target column, use SDTargetEncoder instead."
+            )
+
+        if self.target_encode_cols and y is None:
+            raise ValueError(
+                "target_encode_cols is set but no y was given - "
+                "target encoding needs the target column, call fit(X, y)"
+            )
+
+        X = pd.DataFrame(X)
+        self.columns_ = list(X.columns)
+        # anything not numeric gets treated as categorical - covers "object"
+        # dtype, the newer pandas "str" dtype, and category dtype all at
+        # once instead of checking dtype names one by one (those checks
+        # broke on pandas 3.x, which reports plain strings as dtype "str")
+        self.cat_columns_ = [
+            c for c in self.columns_ if not pd.api.types.is_numeric_dtype(X[c])
+        ]
+
+        self.stats_ = {}
+        for col in self.cat_columns_:
+            if col in self.target_encode_cols:
+                self.stats_[col] = self._fit_target(X[col], y)
+            else:
+                self.stats_[col] = self._fit_one(X[col])
+
+        self.fitted = True
+        return self
+
+    def transform(self, X):
+        if not self.fitted:
+            raise RuntimeError("call fit() before transform()")
+
+        X = pd.DataFrame(X)
+        out = X.copy()
+
+        for col in self.cat_columns_:
+            stat = self.stats_[col]
+
+            if stat["method"] == "onehot":
+                dummies = pd.get_dummies(X[col], prefix=col)
+                # line up columns with what we saw during fit, in case a
+                # category is missing/new in this batch of data
+                dummies = dummies.reindex(columns=stat["dummy_columns"], fill_value=0)
+                out = out.drop(columns=[col])
+                out = pd.concat([out, dummies], axis=1)
+            elif stat["method"] == "target":
+                out[col] = X[col].map(stat["mapping"]).fillna(stat["global_mean"])
+                # unseen categories get the global mean here instead of NaN -
+                # that's the standard fallback for target encoding, "we have
+                # no info about this category so just guess the average"
+            else:
+                out[col] = X[col].map(stat["mapping"])
+                # categories not seen during fit turn into NaN on purpose,
+                # better than silently making up a code for them
+
+        return out
+
+    def fit_transform(self, X, y=None):
+        return self.fit(X, y).transform(X)
+
+    def inverse_transform(self, X):
+        if not self.fitted:
+            raise RuntimeError("call fit() before inverse_transform()")
+
+        X = pd.DataFrame(X).copy()
+        for col in self.cat_columns_:
+            stat = self.stats_[col]
+            if stat["method"] in ("onehot", "target"):
+                raise NotImplementedError(
+                    f"can't auto-reverse a {stat['method']} column ('{col}') through this method"
+                )
+            reverse = {v: k for k, v in stat["mapping"].items()}
+            X[col] = X[col].map(reverse)
+
+        return X
+
+    def summary(self):
+        if not self.fitted:
+            raise RuntimeError("call fit() before summary()")
+
+        rows = []
+        for col in self.columns_:
+            if col in self.stats_:
+                rows.append({
+                    "column": col,
+                    "method": self.stats_[col]["method"],
+                    "categories": self.stats_[col]["n_categories"],
+                })
+            else:
+                rows.append({"column": col, "method": "passthrough", "categories": None})
+
+        return pd.DataFrame(rows)
+
+    def to_column_transformer(self):
+        """
+        builds an sklearn ColumnTransformer that mirrors whatever this
+        encoder decided, in case you'd rather plug it into an sklearn
+        Pipeline instead of calling .transform() directly. needs sklearn.
+
+        note: target-encoded columns get dropped from the ColumnTransformer
+        version since sklearn's transformers don't take y at transform time
+        the way our target method does - if you're using target encoding,
+        just call SDEncoder.transform() directly instead of going through
+        this method for those columns.
+        """
+        if not self.fitted:
+            raise RuntimeError("call fit() before to_column_transformer()")
+
+        try:
+            from sklearn.compose import ColumnTransformer
+            from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
+        except ImportError:
+            raise ImportError("to_column_transformer() needs scikit-learn installed")
+
+        transformers = []
+        for col, stat in self.stats_.items():
+            if stat["method"] == "onehot":
+                transformers.append((f"{col}_onehot", OneHotEncoder(handle_unknown="ignore"), [col]))
+            elif stat["method"] == "target":
+                continue
+            else:
+                order = [list(stat["mapping"].keys())]
+                transformers.append((f"{col}_ordinal", OrdinalEncoder(categories=order), [col]))
+
+        return ColumnTransformer(transformers, remainder="passthrough")
+
+    # internal, don't call directly
+
+    def _fit_one(self, series):
+        col = series.name
+        n_unique = series.nunique(dropna=True)
+
+        if col in self.ordinal_maps:
+            order = self.ordinal_maps[col]
+            mapping = {cat: i for i, cat in enumerate(order)}
+            return {"method": "ordinal", "mapping": mapping, "n_categories": len(order)}
+
+        if n_unique == 2:
+            cats = sorted(series.dropna().unique())
+            mapping = {cats[0]: 0, cats[1]: 1}
+            return {"method": "binary", "mapping": mapping, "n_categories": 2}
+
+        if n_unique <= self.onehot_cutoff:
+            dummies = pd.get_dummies(series, prefix=col)
+            dummy_cols = list(dummies.columns)
+            if self.drop_first and len(dummy_cols) > 1:
+                dummy_cols = dummy_cols[1:]
+            return {"method": "onehot", "dummy_columns": dummy_cols, "n_categories": n_unique}
+
+        # too many categories for onehot to be practical, fall back to ordinal
+        cats = sorted(series.dropna().unique())
+        mapping = {cat: i for i, cat in enumerate(cats)}
+        return {"method": "ordinal", "mapping": mapping, "n_categories": n_unique}
+
+    def _fit_target(self, series, y):
+        y = pd.Series(y).reset_index(drop=True)
+        series = series.reset_index(drop=True)
+
+        global_mean = float(y.mean())
+        smoothing = self.target_smoothing
+
+        grouped = y.groupby(series).agg(["mean", "count"])
+        # blend each category's own mean with the global mean - categories
+        # with few rows lean more on the global mean, categories with lots
+        # of rows lean more on their own mean. classic smoothing trick so
+        # a category that only shows up once or twice doesn't get a wildly
+        # overfit value.
+        smoothed = (grouped["count"] * grouped["mean"] + smoothing * global_mean) / (grouped["count"] + smoothing)
+
+        return {
+            "method": "target",
+            "mapping": smoothed.to_dict(),
+            "global_mean": global_mean,
+            "n_categories": series.nunique(dropna=True),
+        }

sdencoder/target.py

@@ -0,0 +1,66 @@
+import pandas as pd
+
+
+class SDTargetEncoder:
+    """
+    encodes a target column (y) into integer class labels, basically what
+    LabelEncoder does. only meant for a single column, not a full feature
+    dataframe - that's what SDEncoder is for.
+
+    by default it just sorts categories alphabetically and numbers them,
+    which is fine if you don't care which class gets which number. but if
+    your target actually has a real order (low/medium/high) or you care
+    which class ends up as 1 for a binary target (metrics like precision/
+    recall treat 1 as the "positive" class), pass class_order yourself:
+
+    SDTargetEncoder(class_order=["low", "medium", "high"])
+    SDTargetEncoder(class_order=["denied", "approved"])   # approved -> 1
+    """
+
+    def __init__(self, class_order=None):
+        self.class_order = class_order
+        self.mapping_ = None
+        self.classes_ = None
+        self.fitted = False
+
+    def fit(self, y):
+        if isinstance(y, pd.DataFrame):
+            raise TypeError(
+                "SDTargetEncoder wants a single column, not a DataFrame. "
+                "for feature columns use SDEncoder instead."
+            )
+
+        y = pd.Series(y)
+        actual_classes = set(y.dropna().unique())
+
+        if self.class_order is not None:
+            given_classes = set(self.class_order)
+            if given_classes != actual_classes:
+                raise ValueError(
+                    f"class_order {self.class_order} doesn't match the classes "
+                    f"actually found in y: {sorted(actual_classes)}"
+                )
+            self.classes_ = list(self.class_order)
+        else:
+            self.classes_ = sorted(actual_classes)
+
+        self.mapping_ = {c: i for i, c in enumerate(self.classes_)}
+        self.fitted = True
+        return self
+
+    def transform(self, y):
+        if not self.fitted:
+            raise RuntimeError("call fit() before transform()")
+        return pd.Series(y).map(self.mapping_)
+
+    def fit_transform(self, y):
+        return self.fit(y).transform(y)
+
+    def inverse_transform(self, y_encoded):
+        if not self.fitted:
+            raise RuntimeError("call fit() before inverse_transform()")
+        reverse = {v: k for k, v in self.mapping_.items()}
+        return pd.Series(y_encoded).map(reverse)
+
+    def __repr__(self):
+        return f"SDTargetEncoder(classes={self.classes_})"