sdscaler 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sdscaler-0.2.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,155 @@
1
+ Metadata-Version: 2.4
2
+ Name: sdscaler
3
+ Version: 0.2.0
4
+ Summary: auto-scaler that picks standard/minmax/robust scaling per column
5
+ Requires-Python: >=3.8
6
+ Description-Content-Type: text/markdown
7
+ License-File: LICENSE
8
+ Requires-Dist: numpy>=1.20
9
+ Requires-Dist: pandas>=1.2
10
+ Dynamic: license-file
11
+
12
+ # SDScaler
13
+
14
+ Small auto-scaler for tabular data. Instead of manually picking StandardScaler
15
+ vs MinMaxScaler vs RobustScaler for each column, this just looks at the column
16
+ and picks one for you (checks for outliers and skew). You can also just force
17
+ one method if you don't trust the auto part.
18
+
19
+ Made this for a uni project, kept it simple on purpose - no sklearn
20
+ dependency for the core logic (sklearn's only needed if you use
21
+ `to_column_transformer()`), just numpy and pandas.
22
+
23
+ ## Install
24
+
25
+ \```
26
+ cd SDScaler
27
+ pip install -e .
28
+ \```
29
+
30
+ ## Usage
31
+
32
+ \```python
33
+ import pandas as pd
34
+ from sdscaler import SDScaler
35
+
36
+ df = pd.DataFrame({
37
+ "age": [22, 25, 130, 28, 24],
38
+ "income": [30000, 32000, 31000, 500000, 29000],
39
+ })
40
+
41
+ scaler = SDScaler()
42
+ scaled = scaler.fit_transform(df)
43
+
44
+ print(scaled)
45
+ print(scaler.summary()) # which method got used per column
46
+ \```
47
+
48
+ Force one method instead of auto:
49
+
50
+ \```python
51
+ scaler = SDScaler(method="standard") # or "minmax" / "robust" / "log"
52
+ \```
53
+
54
+ Undo the scaling later:
55
+
56
+ \```python
57
+ scaler.inverse_transform(scaled)
58
+ \```
59
+
60
+ Remember to fit only on your training data, then just `.transform()` the
61
+ test set with those same stats - don't fit_transform both separately or
62
+ you'll leak info / get mismatched scales.
63
+
64
+ \```python
65
+ train_scaled = scaler.fit_transform(train_df)
66
+ test_scaled = scaler.transform(test_df)
67
+ \```
68
+
69
+ ## How the auto part decides
70
+
71
+ For each numeric column:
72
+ - if more than 5% of points are outliers (1.5*IQR rule) -> robust
73
+ - else if it's pretty skewed (skew > 1) and strictly positive -> log
74
+ - else if it's pretty skewed but has zero/negative values -> minmax
75
+ - otherwise -> standard
76
+
77
+ Both cutoffs can be changed:
78
+
79
+ \```python
80
+ SDScaler(outlier_cutoff=0.1, skew_cutoff=1.5)
81
+ \```
82
+
83
+ Heads up - on small datasets one weird value can be enough to trip the
84
+ outlier check even if the column mostly looks normal, so don't be surprised
85
+ if a small demo dataset gets more "robust" columns than you'd expect.
86
+
87
+ ## Forcing log scaling on specific columns
88
+
89
+ \```python
90
+ scaler = SDScaler(log_cols=["income"])
91
+ \```
92
+
93
+ This scales `income` with log regardless of what auto-pick would've chosen,
94
+ everything else still gets decided normally. Log scaling shifts the column
95
+ to be positive first if it isn't already, so it won't blow up on zeros or
96
+ negative values, but it's really meant for stuff that's naturally always
97
+ positive and skewed (income, population, prices, that kind of thing).
98
+
99
+ ## Non-numeric columns
100
+
101
+ If you pass a dataframe that still has a categorical/text column in it (like
102
+ forgot to encode it, or it's a column you're not scaling on purpose), SDScaler
103
+ just leaves it alone instead of erroring out. Only numeric columns get touched.
104
+
105
+ \```python
106
+ scaler.summary()
107
+ # column method
108
+ # 0 age standard
109
+ # 1 city passthrough (not numeric)
110
+ \```
111
+
112
+ ## Missing values
113
+
114
+ If a column has NaN in it, `.fit()` still works and warns you about which
115
+ columns have missing data. The stats (mean, median, etc.) get computed
116
+ ignoring the NaNs, but the scaled output will still have NaN in those exact
117
+ spots - didn't want to silently fill anything in without you knowing.
118
+
119
+ ## Using it with sklearn Pipelines
120
+
121
+ \```python
122
+ scaler = SDScaler().fit(df)
123
+ ct = scaler.to_column_transformer()
124
+ \```
125
+
126
+ Builds an sklearn `ColumnTransformer` using the same decisions SDScaler
127
+ already made, for anyone who'd rather have it inside a Pipeline instead of
128
+ calling `.transform()` directly. Non-numeric columns get passed through here
129
+ too. Needs scikit-learn installed, only used if you actually call this.
130
+
131
+ ## Files
132
+
133
+ \```
134
+ sdscaler/
135
+ scaler.py - the SDScaler class
136
+ helpers.py - skew/outlier detection logic
137
+ examples/
138
+ example_usage.py
139
+ tests/
140
+ test_scaler.py
141
+ \```
142
+
143
+ ## Running tests
144
+
145
+ \```
146
+ pip install pytest
147
+ pytest tests/
148
+ \```
149
+
150
+ ## Todo / ideas
151
+
152
+ - frequency encoding style option for very high-cardinality numeric IDs
153
+ - maybe a plot function to compare before/after distributions
154
+ - log scaling on negative-heavy data currently just shifts everything, could
155
+ be smarter about it for columns that are mostly negative
@@ -0,0 +1,144 @@
1
+ # SDScaler
2
+
3
+ Small auto-scaler for tabular data. Instead of manually picking StandardScaler
4
+ vs MinMaxScaler vs RobustScaler for each column, this just looks at the column
5
+ and picks one for you (checks for outliers and skew). You can also just force
6
+ one method if you don't trust the auto part.
7
+
8
+ Made this for a uni project, kept it simple on purpose - no sklearn
9
+ dependency for the core logic (sklearn's only needed if you use
10
+ `to_column_transformer()`), just numpy and pandas.
11
+
12
+ ## Install
13
+
14
+ \```
15
+ cd SDScaler
16
+ pip install -e .
17
+ \```
18
+
19
+ ## Usage
20
+
21
+ \```python
22
+ import pandas as pd
23
+ from sdscaler import SDScaler
24
+
25
+ df = pd.DataFrame({
26
+ "age": [22, 25, 130, 28, 24],
27
+ "income": [30000, 32000, 31000, 500000, 29000],
28
+ })
29
+
30
+ scaler = SDScaler()
31
+ scaled = scaler.fit_transform(df)
32
+
33
+ print(scaled)
34
+ print(scaler.summary()) # which method got used per column
35
+ \```
36
+
37
+ Force one method instead of auto:
38
+
39
+ \```python
40
+ scaler = SDScaler(method="standard") # or "minmax" / "robust" / "log"
41
+ \```
42
+
43
+ Undo the scaling later:
44
+
45
+ \```python
46
+ scaler.inverse_transform(scaled)
47
+ \```
48
+
49
+ Remember to fit only on your training data, then just `.transform()` the
50
+ test set with those same stats - don't fit_transform both separately or
51
+ you'll leak info / get mismatched scales.
52
+
53
+ \```python
54
+ train_scaled = scaler.fit_transform(train_df)
55
+ test_scaled = scaler.transform(test_df)
56
+ \```
57
+
58
+ ## How the auto part decides
59
+
60
+ For each numeric column:
61
+ - if more than 5% of points are outliers (1.5*IQR rule) -> robust
62
+ - else if it's pretty skewed (skew > 1) and strictly positive -> log
63
+ - else if it's pretty skewed but has zero/negative values -> minmax
64
+ - otherwise -> standard
65
+
66
+ Both cutoffs can be changed:
67
+
68
+ \```python
69
+ SDScaler(outlier_cutoff=0.1, skew_cutoff=1.5)
70
+ \```
71
+
72
+ Heads up - on small datasets one weird value can be enough to trip the
73
+ outlier check even if the column mostly looks normal, so don't be surprised
74
+ if a small demo dataset gets more "robust" columns than you'd expect.
75
+
76
+ ## Forcing log scaling on specific columns
77
+
78
+ \```python
79
+ scaler = SDScaler(log_cols=["income"])
80
+ \```
81
+
82
+ This scales `income` with log regardless of what auto-pick would've chosen,
83
+ everything else still gets decided normally. Log scaling shifts the column
84
+ to be positive first if it isn't already, so it won't blow up on zeros or
85
+ negative values, but it's really meant for stuff that's naturally always
86
+ positive and skewed (income, population, prices, that kind of thing).
87
+
88
+ ## Non-numeric columns
89
+
90
+ If you pass a dataframe that still has a categorical/text column in it (like
91
+ forgot to encode it, or it's a column you're not scaling on purpose), SDScaler
92
+ just leaves it alone instead of erroring out. Only numeric columns get touched.
93
+
94
+ \```python
95
+ scaler.summary()
96
+ # column method
97
+ # 0 age standard
98
+ # 1 city passthrough (not numeric)
99
+ \```
100
+
101
+ ## Missing values
102
+
103
+ If a column has NaN in it, `.fit()` still works and warns you about which
104
+ columns have missing data. The stats (mean, median, etc.) get computed
105
+ ignoring the NaNs, but the scaled output will still have NaN in those exact
106
+ spots - didn't want to silently fill anything in without you knowing.
107
+
108
+ ## Using it with sklearn Pipelines
109
+
110
+ \```python
111
+ scaler = SDScaler().fit(df)
112
+ ct = scaler.to_column_transformer()
113
+ \```
114
+
115
+ Builds an sklearn `ColumnTransformer` using the same decisions SDScaler
116
+ already made, for anyone who'd rather have it inside a Pipeline instead of
117
+ calling `.transform()` directly. Non-numeric columns get passed through here
118
+ too. Needs scikit-learn installed, only used if you actually call this.
119
+
120
+ ## Files
121
+
122
+ \```
123
+ sdscaler/
124
+ scaler.py - the SDScaler class
125
+ helpers.py - skew/outlier detection logic
126
+ examples/
127
+ example_usage.py
128
+ tests/
129
+ test_scaler.py
130
+ \```
131
+
132
+ ## Running tests
133
+
134
+ \```
135
+ pip install pytest
136
+ pytest tests/
137
+ \```
138
+
139
+ ## Todo / ideas
140
+
141
+ - frequency encoding style option for very high-cardinality numeric IDs
142
+ - maybe a plot function to compare before/after distributions
143
+ - log scaling on negative-heavy data currently just shifts everything, could
144
+ be smarter about it for columns that are mostly negative
@@ -0,0 +1,17 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "sdscaler"
7
+ version = "0.2.0"
8
+ description = "auto-scaler that picks standard/minmax/robust scaling per column"
9
+ readme = "README.md"
10
+ requires-python = ">=3.8"
11
+ dependencies = [
12
+ "numpy>=1.20",
13
+ "pandas>=1.2",
14
+ ]
15
+
16
+ [tool.setuptools.packages.find]
17
+ include = ["sdscaler*"]
@@ -0,0 +1,3 @@
1
+ from .scaler import SDScaler
2
+
3
+ __version__ = "0.2.0"
@@ -0,0 +1,46 @@
1
+ # figures out which scaling method fits a column best
2
+ # just checks for outliers and skew, nothing fancy
3
+
4
+ import numpy as np
5
+
6
+
7
+ def skewness(x):
8
+ x = np.asarray(x, dtype=float)
9
+ if len(x) < 3:
10
+ return 0.0
11
+
12
+ mean = np.nanmean(x)
13
+ std = np.nanstd(x)
14
+ if std == 0:
15
+ return 0.0
16
+
17
+ return np.nanmean(((x - mean) / std) ** 3)
18
+
19
+
20
+ def outlier_frac(x):
21
+ # classic 1.5*IQR rule from stats class
22
+ x = np.asarray(x, dtype=float)
23
+ q1, q3 = np.nanpercentile(x, [25, 75])
24
+ iqr = q3 - q1
25
+
26
+ if iqr == 0:
27
+ return 0.0
28
+
29
+ low = q1 - 1.5 * iqr
30
+ high = q3 + 1.5 * iqr
31
+ outliers = (x < low) | (x > high)
32
+ return np.nansum(outliers) / len(x)
33
+
34
+
35
+ def pick_method(x, skew_cutoff=1.0, outlier_cutoff=0.05):
36
+ if outlier_frac(x) > outlier_cutoff:
37
+ return "robust"
38
+
39
+ if abs(skewness(x)) > skew_cutoff:
40
+ # log only makes sense if the column is strictly positive, otherwise
41
+ # fall back to minmax like before
42
+ if np.nanmin(x) > 0:
43
+ return "log"
44
+ return "minmax"
45
+
46
+ return "standard"
@@ -0,0 +1,262 @@
1
+ import warnings
2
+
3
+ import numpy as np
4
+ import pandas as pd
5
+
6
+ from .helpers import pick_method
7
+
8
+ METHODS = ("auto", "standard", "minmax", "robust", "log")
9
+
10
+
11
+ class SDScaler:
12
+ """
13
+ basically StandardScaler/MinMaxScaler/RobustScaler but it picks one
14
+ for you per column instead of you having to guess which one to use.
15
+
16
+ scaler = SDScaler()
17
+ scaled = scaler.fit_transform(df)
18
+ scaler.summary() # shows what got picked for each column
19
+
20
+ set method to "standard"/"minmax"/"robust"/"log" if you want to force
21
+ one instead of letting it decide (default is "auto")
22
+
23
+ pass log_cols if you want specific columns log-scaled regardless of
24
+ what auto-pick would've chosen - useful for stuff like income that's
25
+ always heavily skewed. auto-pick will also choose log on its own for
26
+ skewed, strictly-positive columns, log_cols just forces it.
27
+
28
+ if X is a dataframe, non-numeric columns get left alone (passthrough)
29
+ instead of crashing - only numeric columns actually get scaled.
30
+ """
31
+
32
+ def __init__(self, method="auto", outlier_cutoff=0.05, skew_cutoff=1.0, log_cols=None):
33
+ if method not in METHODS:
34
+ raise ValueError(f"method has to be one of {METHODS}, got '{method}'")
35
+
36
+ self.method = method
37
+ self.outlier_cutoff = outlier_cutoff
38
+ self.skew_cutoff = skew_cutoff
39
+ self.log_cols = log_cols or []
40
+
41
+ self.stats_ = {}
42
+ self.columns_ = None
43
+ self.numeric_columns_ = None
44
+ self.is_df = False
45
+ self.fitted = False
46
+
47
+ def fit(self, X):
48
+ self.is_df = isinstance(X, pd.DataFrame)
49
+
50
+ if self.is_df:
51
+ self.columns_ = list(X.columns)
52
+ self.numeric_columns_ = [
53
+ c for c in self.columns_ if pd.api.types.is_numeric_dtype(X[c])
54
+ ]
55
+ nan_cols = [c for c in self.numeric_columns_ if X[c].isna().any()]
56
+ if nan_cols:
57
+ warnings.warn(
58
+ f"these columns have missing values, stats were computed "
59
+ f"ignoring them but the scaled output will still have NaN "
60
+ f"in those spots: {nan_cols}"
61
+ )
62
+ data_lookup = {c: X[c].to_numpy(dtype=float) for c in self.numeric_columns_}
63
+ else:
64
+ X = np.asarray(X, dtype=float)
65
+ if X.ndim == 1:
66
+ X = X.reshape(-1, 1)
67
+ self.columns_ = list(range(X.shape[1]))
68
+ self.numeric_columns_ = list(self.columns_)
69
+ data_lookup = {c: X[:, i] for i, c in enumerate(self.columns_)}
70
+
71
+ self.stats_ = {}
72
+ for col in self.numeric_columns_:
73
+ col_data = data_lookup[col]
74
+ if col in self.log_cols:
75
+ method = "log"
76
+ elif self.method == "auto":
77
+ method = pick_method(col_data, self.skew_cutoff, self.outlier_cutoff)
78
+ else:
79
+ method = self.method
80
+ self.stats_[col] = self._fit_one(col_data, method)
81
+
82
+ self.fitted = True
83
+ return self
84
+
85
+ def transform(self, X):
86
+ if not self.fitted:
87
+ raise RuntimeError("call fit() before transform()")
88
+
89
+ if self.is_df:
90
+ out = X.copy()
91
+ for col in self.numeric_columns_:
92
+ out[col] = self._apply_one(X[col].to_numpy(dtype=float), self.stats_[col])
93
+ return out
94
+
95
+ X = np.asarray(X, dtype=float)
96
+ squeeze = X.ndim == 1
97
+ if squeeze:
98
+ X = X.reshape(-1, 1)
99
+
100
+ out = np.empty_like(X, dtype=float)
101
+ for i, col in enumerate(self.numeric_columns_):
102
+ out[:, i] = self._apply_one(X[:, i], self.stats_[col])
103
+
104
+ return out.ravel() if squeeze else out
105
+
106
+ def fit_transform(self, X):
107
+ return self.fit(X).transform(X)
108
+
109
+ def inverse_transform(self, X):
110
+ if not self.fitted:
111
+ raise RuntimeError("call fit() before inverse_transform()")
112
+
113
+ if self.is_df:
114
+ out = X.copy()
115
+ for col in self.numeric_columns_:
116
+ out[col] = self._undo_one(X[col].to_numpy(dtype=float), self.stats_[col])
117
+ return out
118
+
119
+ X = np.asarray(X, dtype=float)
120
+ squeeze = X.ndim == 1
121
+ if squeeze:
122
+ X = X.reshape(-1, 1)
123
+
124
+ out = np.empty_like(X, dtype=float)
125
+ for i, col in enumerate(self.numeric_columns_):
126
+ out[:, i] = self._undo_one(X[:, i], self.stats_[col])
127
+
128
+ return out.ravel() if squeeze else out
129
+
130
+ def summary(self):
131
+ if not self.fitted:
132
+ raise RuntimeError("call fit() before summary()")
133
+
134
+ rows = []
135
+ for col in self.columns_:
136
+ if col in self.stats_:
137
+ rows.append({"column": col, "method": self.stats_[col]["method"]})
138
+ else:
139
+ rows.append({"column": col, "method": "passthrough (not numeric)"})
140
+ return pd.DataFrame(rows)
141
+
142
+ def to_column_transformer(self):
143
+ """
144
+ builds an sklearn ColumnTransformer that mirrors whatever this
145
+ scaler decided, in case you'd rather plug it into an sklearn
146
+ Pipeline instead of calling .transform() directly. needs sklearn.
147
+ non-numeric columns get passed through untouched, same as here.
148
+ """
149
+ if not self.fitted:
150
+ raise RuntimeError("call fit() before to_column_transformer()")
151
+
152
+ try:
153
+ from sklearn.compose import ColumnTransformer
154
+ from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
155
+ except ImportError:
156
+ raise ImportError("to_column_transformer() needs scikit-learn installed")
157
+
158
+ transformers = []
159
+ for col, stat in self.stats_.items():
160
+ if stat["method"] == "standard":
161
+ transformers.append((f"{col}_standard", StandardScaler(), [col]))
162
+ elif stat["method"] == "minmax":
163
+ transformers.append((f"{col}_minmax", MinMaxScaler(), [col]))
164
+ elif stat["method"] == "robust":
165
+ transformers.append((f"{col}_robust", RobustScaler(), [col]))
166
+ elif stat["method"] == "log":
167
+ transformers.append((f"{col}_log", _LogScaler(), [col]))
168
+
169
+ return ColumnTransformer(transformers, remainder="passthrough")
170
+
171
+ # internal, don't call directly
172
+
173
+ def _fit_one(self, x, method):
174
+ if method == "standard":
175
+ mean, std = np.nanmean(x), np.nanstd(x)
176
+ return {"method": "standard", "mean": mean, "std": std or 1.0}
177
+
178
+ if method == "minmax":
179
+ lo, hi = np.nanmin(x), np.nanmax(x)
180
+ span = hi - lo
181
+ return {"method": "minmax", "min": lo, "max": hi, "span": span or 1.0}
182
+
183
+ if method == "robust":
184
+ med = np.nanmedian(x)
185
+ q1, q3 = np.nanpercentile(x, [25, 75])
186
+ iqr = q3 - q1
187
+ return {"method": "robust", "median": med, "iqr": iqr or 1.0}
188
+
189
+ if method == "log":
190
+ m = np.nanmin(x)
191
+ # shift so everything's positive before taking the log - if the
192
+ # column's already all positive this is just offset=0 and does
193
+ # nothing
194
+ offset = (1 - m) if m <= 0 else 0.0
195
+ shifted = np.clip(x + offset, 1e-6, None)
196
+ logged = np.log(shifted)
197
+ mean, std = np.nanmean(logged), np.nanstd(logged)
198
+ return {"method": "log", "offset": offset, "mean": mean, "std": std or 1.0}
199
+
200
+ raise ValueError(f"unknown method {method}")
201
+
202
+ def _apply_one(self, x, s):
203
+ x = np.asarray(x, dtype=float)
204
+ if s["method"] == "standard":
205
+ return (x - s["mean"]) / s["std"]
206
+ if s["method"] == "minmax":
207
+ return (x - s["min"]) / s["span"]
208
+ if s["method"] == "robust":
209
+ return (x - s["median"]) / s["iqr"]
210
+ if s["method"] == "log":
211
+ shifted = np.clip(x + s["offset"], 1e-6, None)
212
+ logged = np.log(shifted)
213
+ return (logged - s["mean"]) / s["std"]
214
+
215
+ def _undo_one(self, x, s):
216
+ x = np.asarray(x, dtype=float)
217
+ if s["method"] == "standard":
218
+ return x * s["std"] + s["mean"]
219
+ if s["method"] == "minmax":
220
+ return x * s["span"] + s["min"]
221
+ if s["method"] == "robust":
222
+ return x * s["iqr"] + s["median"]
223
+ if s["method"] == "log":
224
+ logged = x * s["std"] + s["mean"]
225
+ shifted = np.exp(logged)
226
+ return shifted - s["offset"]
227
+
228
+ def __repr__(self):
229
+ return f"SDScaler(method='{self.method}')"
230
+
231
+
232
+ class _LogScaler:
233
+ """
234
+ small helper class used by to_column_transformer() for log-method
235
+ columns - sklearn's ColumnTransformer needs something with its own
236
+ fit/transform, this is just our log logic wrapped up that way.
237
+ """
238
+
239
+ def fit(self, X, y=None):
240
+ x = np.asarray(X, dtype=float).ravel()
241
+ m = np.nanmin(x)
242
+ self.offset_ = (1 - m) if m <= 0 else 0.0
243
+ shifted = np.clip(x + self.offset_, 1e-6, None)
244
+ logged = np.log(shifted)
245
+ self.mean_ = np.nanmean(logged)
246
+ self.std_ = np.nanstd(logged) or 1.0
247
+ return self
248
+
249
+ def transform(self, X):
250
+ x = np.asarray(X, dtype=float).ravel()
251
+ shifted = np.clip(x + self.offset_, 1e-6, None)
252
+ logged = np.log(shifted)
253
+ return ((logged - self.mean_) / self.std_).reshape(-1, 1)
254
+
255
+ def fit_transform(self, X, y=None):
256
+ return self.fit(X, y).transform(X)
257
+
258
+ def get_params(self, deep=True):
259
+ return {}
260
+
261
+ def set_params(self, **params):
262
+ return self
@@ -0,0 +1,155 @@
1
+ Metadata-Version: 2.4
2
+ Name: sdscaler
3
+ Version: 0.2.0
4
+ Summary: auto-scaler that picks standard/minmax/robust scaling per column
5
+ Requires-Python: >=3.8
6
+ Description-Content-Type: text/markdown
7
+ License-File: LICENSE
8
+ Requires-Dist: numpy>=1.20
9
+ Requires-Dist: pandas>=1.2
10
+ Dynamic: license-file
11
+
12
+ # SDScaler
13
+
14
+ Small auto-scaler for tabular data. Instead of manually picking StandardScaler
15
+ vs MinMaxScaler vs RobustScaler for each column, this just looks at the column
16
+ and picks one for you (checks for outliers and skew). You can also just force
17
+ one method if you don't trust the auto part.
18
+
19
+ Made this for a uni project, kept it simple on purpose - no sklearn
20
+ dependency for the core logic (sklearn's only needed if you use
21
+ `to_column_transformer()`), just numpy and pandas.
22
+
23
+ ## Install
24
+
25
+ \```
26
+ cd SDScaler
27
+ pip install -e .
28
+ \```
29
+
30
+ ## Usage
31
+
32
+ \```python
33
+ import pandas as pd
34
+ from sdscaler import SDScaler
35
+
36
+ df = pd.DataFrame({
37
+ "age": [22, 25, 130, 28, 24],
38
+ "income": [30000, 32000, 31000, 500000, 29000],
39
+ })
40
+
41
+ scaler = SDScaler()
42
+ scaled = scaler.fit_transform(df)
43
+
44
+ print(scaled)
45
+ print(scaler.summary()) # which method got used per column
46
+ \```
47
+
48
+ Force one method instead of auto:
49
+
50
+ \```python
51
+ scaler = SDScaler(method="standard") # or "minmax" / "robust" / "log"
52
+ \```
53
+
54
+ Undo the scaling later:
55
+
56
+ \```python
57
+ scaler.inverse_transform(scaled)
58
+ \```
59
+
60
+ Remember to fit only on your training data, then just `.transform()` the
61
+ test set with those same stats - don't fit_transform both separately or
62
+ you'll leak info / get mismatched scales.
63
+
64
+ \```python
65
+ train_scaled = scaler.fit_transform(train_df)
66
+ test_scaled = scaler.transform(test_df)
67
+ \```
68
+
69
+ ## How the auto part decides
70
+
71
+ For each numeric column:
72
+ - if more than 5% of points are outliers (1.5*IQR rule) -> robust
73
+ - else if it's pretty skewed (skew > 1) and strictly positive -> log
74
+ - else if it's pretty skewed but has zero/negative values -> minmax
75
+ - otherwise -> standard
76
+
77
+ Both cutoffs can be changed:
78
+
79
+ \```python
80
+ SDScaler(outlier_cutoff=0.1, skew_cutoff=1.5)
81
+ \```
82
+
83
+ Heads up - on small datasets one weird value can be enough to trip the
84
+ outlier check even if the column mostly looks normal, so don't be surprised
85
+ if a small demo dataset gets more "robust" columns than you'd expect.
86
+
87
+ ## Forcing log scaling on specific columns
88
+
89
+ \```python
90
+ scaler = SDScaler(log_cols=["income"])
91
+ \```
92
+
93
+ This scales `income` with log regardless of what auto-pick would've chosen,
94
+ everything else still gets decided normally. Log scaling shifts the column
95
+ to be positive first if it isn't already, so it won't blow up on zeros or
96
+ negative values, but it's really meant for stuff that's naturally always
97
+ positive and skewed (income, population, prices, that kind of thing).
98
+
99
+ ## Non-numeric columns
100
+
101
+ If you pass a dataframe that still has a categorical/text column in it (like
102
+ forgot to encode it, or it's a column you're not scaling on purpose), SDScaler
103
+ just leaves it alone instead of erroring out. Only numeric columns get touched.
104
+
105
+ \```python
106
+ scaler.summary()
107
+ # column method
108
+ # 0 age standard
109
+ # 1 city passthrough (not numeric)
110
+ \```
111
+
112
+ ## Missing values
113
+
114
+ If a column has NaN in it, `.fit()` still works and warns you about which
115
+ columns have missing data. The stats (mean, median, etc.) get computed
116
+ ignoring the NaNs, but the scaled output will still have NaN in those exact
117
+ spots - didn't want to silently fill anything in without you knowing.
118
+
119
+ ## Using it with sklearn Pipelines
120
+
121
+ \```python
122
+ scaler = SDScaler().fit(df)
123
+ ct = scaler.to_column_transformer()
124
+ \```
125
+
126
+ Builds an sklearn `ColumnTransformer` using the same decisions SDScaler
127
+ already made, for anyone who'd rather have it inside a Pipeline instead of
128
+ calling `.transform()` directly. Non-numeric columns get passed through here
129
+ too. Needs scikit-learn installed, only used if you actually call this.
130
+
131
+ ## Files
132
+
133
+ \```
134
+ sdscaler/
135
+ scaler.py - the SDScaler class
136
+ helpers.py - skew/outlier detection logic
137
+ examples/
138
+ example_usage.py
139
+ tests/
140
+ test_scaler.py
141
+ \```
142
+
143
+ ## Running tests
144
+
145
+ \```
146
+ pip install pytest
147
+ pytest tests/
148
+ \```
149
+
150
+ ## Todo / ideas
151
+
152
+ - frequency encoding style option for very high-cardinality numeric IDs
153
+ - maybe a plot function to compare before/after distributions
154
+ - log scaling on negative-heavy data currently just shifts everything, could
155
+ be smarter about it for columns that are mostly negative
@@ -0,0 +1,12 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ sdscaler/__init__.py
5
+ sdscaler/helpers.py
6
+ sdscaler/scaler.py
7
+ sdscaler.egg-info/PKG-INFO
8
+ sdscaler.egg-info/SOURCES.txt
9
+ sdscaler.egg-info/dependency_links.txt
10
+ sdscaler.egg-info/requires.txt
11
+ sdscaler.egg-info/top_level.txt
12
+ tests/test_scaler.py
@@ -0,0 +1,2 @@
1
+ numpy>=1.20
2
+ pandas>=1.2
@@ -0,0 +1 @@
1
+ sdscaler
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,63 @@
1
+ import numpy as np
2
+ import pandas as pd
3
+ import pytest
4
+
5
+ from sdscaler import SDScaler
6
+
7
+
8
+ def sample_df():
9
+ return pd.DataFrame({
10
+ "normal": [10, 12, 11, 13, 9, 10, 12, 11],
11
+ "skewed": [1, 2, 1, 2, 1, 2, 1, 100],
12
+ "outliers": [5, 6, 5, 7, 6, 5, 6, 500],
13
+ })
14
+
15
+
16
+ def test_shape_stays_same():
17
+ df = sample_df()
18
+ out = SDScaler().fit_transform(df)
19
+ assert out.shape == df.shape
20
+
21
+
22
+ def test_picks_valid_methods():
23
+ df = sample_df()
24
+ s = SDScaler().fit(df)
25
+ for stat in s.stats_.values():
26
+ assert stat["method"] in ("standard", "minmax", "robust")
27
+
28
+
29
+ def test_outlier_col_goes_robust():
30
+ df = sample_df()
31
+ s = SDScaler().fit(df)
32
+ assert s.stats_["outliers"]["method"] == "robust"
33
+
34
+
35
+ def test_roundtrip():
36
+ df = sample_df()
37
+ s = SDScaler()
38
+ scaled = s.fit_transform(df)
39
+ back = s.inverse_transform(scaled)
40
+ np.testing.assert_allclose(back.values, df.values, rtol=1e-6)
41
+
42
+
43
+ def test_forced_method():
44
+ df = sample_df()
45
+ s = SDScaler(method="standard").fit(df)
46
+ for stat in s.stats_.values():
47
+ assert stat["method"] == "standard"
48
+
49
+
50
+ def test_transform_needs_fit_first():
51
+ with pytest.raises(RuntimeError):
52
+ SDScaler().transform(sample_df())
53
+
54
+
55
+ def test_bad_method_name():
56
+ with pytest.raises(ValueError):
57
+ SDScaler(method="whatever")
58
+
59
+
60
+ def test_works_with_numpy():
61
+ arr = np.array([[1, 2], [3, 4], [5, 100]])
62
+ out = SDScaler().fit_transform(arr)
63
+ assert out.shape == (3, 2)