edaprep 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edaprep-0.1.0/LICENSE +21 -0
- edaprep-0.1.0/PKG-INFO +306 -0
- edaprep-0.1.0/README.md +260 -0
- edaprep-0.1.0/pyproject.toml +97 -0
- edaprep-0.1.0/setup.cfg +4 -0
- edaprep-0.1.0/src/edaprep/__init__.py +188 -0
- edaprep-0.1.0/src/edaprep/_version.py +1 -0
- edaprep-0.1.0/src/edaprep/backends/__init__.py +11 -0
- edaprep-0.1.0/src/edaprep/backends/base.py +167 -0
- edaprep-0.1.0/src/edaprep/backends/pandas_backend.py +128 -0
- edaprep-0.1.0/src/edaprep/config.py +504 -0
- edaprep-0.1.0/src/edaprep/core/__init__.py +7 -0
- edaprep-0.1.0/src/edaprep/core/base.py +314 -0
- edaprep-0.1.0/src/edaprep/core/context.py +81 -0
- edaprep-0.1.0/src/edaprep/core/journal.py +244 -0
- edaprep-0.1.0/src/edaprep/core/pipeline.py +587 -0
- edaprep-0.1.0/src/edaprep/eda/__init__.py +22 -0
- edaprep-0.1.0/src/edaprep/eda/analyzer.py +486 -0
- edaprep-0.1.0/src/edaprep/eda/categorical.py +100 -0
- edaprep-0.1.0/src/edaprep/eda/correlation.py +183 -0
- edaprep-0.1.0/src/edaprep/eda/numerical.py +86 -0
- edaprep-0.1.0/src/edaprep/eda/outliers.py +99 -0
- edaprep-0.1.0/src/edaprep/eda/target.py +214 -0
- edaprep-0.1.0/src/edaprep/exceptions.py +161 -0
- edaprep-0.1.0/src/edaprep/planning/__init__.py +16 -0
- edaprep-0.1.0/src/edaprep/planning/decisions.py +358 -0
- edaprep-0.1.0/src/edaprep/planning/planner.py +364 -0
- edaprep-0.1.0/src/edaprep/planning/rules.py +819 -0
- edaprep-0.1.0/src/edaprep/preprocessing/__init__.py +67 -0
- edaprep-0.1.0/src/edaprep/preprocessing/casting.py +260 -0
- edaprep-0.1.0/src/edaprep/preprocessing/datetime_features.py +208 -0
- edaprep-0.1.0/src/edaprep/preprocessing/duplicates.py +170 -0
- edaprep-0.1.0/src/edaprep/preprocessing/encoding.py +804 -0
- edaprep-0.1.0/src/edaprep/preprocessing/missing.py +363 -0
- edaprep-0.1.0/src/edaprep/preprocessing/outliers.py +569 -0
- edaprep-0.1.0/src/edaprep/preprocessing/scaling.py +228 -0
- edaprep-0.1.0/src/edaprep/preprocessing/selection.py +468 -0
- edaprep-0.1.0/src/edaprep/preprocessing/text.py +124 -0
- edaprep-0.1.0/src/edaprep/preprocessing/transformations.py +379 -0
- edaprep-0.1.0/src/edaprep/profiling/__init__.py +19 -0
- edaprep-0.1.0/src/edaprep/profiling/column_types.py +498 -0
- edaprep-0.1.0/src/edaprep/profiling/profiler.py +1095 -0
- edaprep-0.1.0/src/edaprep/profiling/quality.py +357 -0
- edaprep-0.1.0/src/edaprep/profiling/statistics.py +516 -0
- edaprep-0.1.0/src/edaprep/py.typed +0 -0
- edaprep-0.1.0/src/edaprep/reporting/__init__.py +5 -0
- edaprep-0.1.0/src/edaprep/reporting/html.py +173 -0
- edaprep-0.1.0/src/edaprep/reporting/report.py +317 -0
- edaprep-0.1.0/src/edaprep/types.py +189 -0
- edaprep-0.1.0/src/edaprep/visualization/__init__.py +33 -0
- edaprep-0.1.0/src/edaprep/visualization/plots.py +330 -0
- edaprep-0.1.0/src/edaprep.egg-info/PKG-INFO +306 -0
- edaprep-0.1.0/src/edaprep.egg-info/SOURCES.txt +61 -0
- edaprep-0.1.0/src/edaprep.egg-info/dependency_links.txt +1 -0
- edaprep-0.1.0/src/edaprep.egg-info/requires.txt +24 -0
- edaprep-0.1.0/src/edaprep.egg-info/top_level.txt +1 -0
- edaprep-0.1.0/tests/test_column_types.py +211 -0
- edaprep-0.1.0/tests/test_eda.py +420 -0
- edaprep-0.1.0/tests/test_leakage.py +385 -0
- edaprep-0.1.0/tests/test_pipeline.py +606 -0
- edaprep-0.1.0/tests/test_profiler.py +288 -0
- edaprep-0.1.0/tests/test_statistics.py +243 -0
- edaprep-0.1.0/tests/test_transformers.py +917 -0
edaprep-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 bijay
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
edaprep-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: edaprep
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Transparent, leakage-safe EDA and ML preprocessing with an explainable planner.
|
|
5
|
+
Author-email: bijay <bijaybeezoe@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Documentation, https://github.com/bijay-odyssey/edaprep#readme
|
|
8
|
+
Project-URL: Source, https://github.com/bijay-odyssey/edaprep
|
|
9
|
+
Project-URL: Changelog, https://github.com/bijay-odyssey/edaprep/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Issues, https://github.com/bijay-odyssey/edaprep/issues
|
|
11
|
+
Keywords: eda,preprocessing,data-cleaning,machine-learning,pandas
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: numpy>=1.21
|
|
27
|
+
Requires-Dist: pandas>=1.5
|
|
28
|
+
Requires-Dist: scipy>=1.7
|
|
29
|
+
Provides-Extra: visualization
|
|
30
|
+
Requires-Dist: matplotlib>=3.5; extra == "visualization"
|
|
31
|
+
Provides-Extra: advanced
|
|
32
|
+
Requires-Dist: scikit-learn>=1.1; extra == "advanced"
|
|
33
|
+
Provides-Extra: arrow
|
|
34
|
+
Requires-Dist: pyarrow>=10.0; extra == "arrow"
|
|
35
|
+
Provides-Extra: all
|
|
36
|
+
Requires-Dist: matplotlib>=3.5; extra == "all"
|
|
37
|
+
Requires-Dist: scikit-learn>=1.1; extra == "all"
|
|
38
|
+
Requires-Dist: pyarrow>=10.0; extra == "all"
|
|
39
|
+
Provides-Extra: dev
|
|
40
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
41
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
42
|
+
Requires-Dist: hypothesis>=6.60; extra == "dev"
|
|
43
|
+
Requires-Dist: scikit-learn>=1.1; extra == "dev"
|
|
44
|
+
Requires-Dist: matplotlib>=3.5; extra == "dev"
|
|
45
|
+
Dynamic: license-file
|
|
46
|
+
|
|
47
|
+
# edaprep
|
|
48
|
+
|
|
49
|
+
[](https://github.com/bijay-odyssey/edaprep/actions/workflows/ci.yml)
|
|
50
|
+
[](https://pypi.org/project/edaprep/)
|
|
51
|
+
[](LICENSE)
|
|
52
|
+
|
|
53
|
+
Transparent, leakage-safe EDA and ML preprocessing, with an explainable planner.
|
|
54
|
+
|
|
55
|
+
`edaprep` looks at a dataset, works out which preprocessing operations actually apply
|
|
56
|
+
to it, tells you what it intends to do and why, and then does it — fitting every
|
|
57
|
+
statistic on the training data alone.
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
import edaprep
|
|
61
|
+
|
|
62
|
+
pipe = edaprep.AutoPipeline(target="churn", model_family="tree", random_state=42)
|
|
63
|
+
pipe.fit(train_df)
|
|
64
|
+
pipe.explain()
|
|
65
|
+
|
|
66
|
+
X_train = pipe.transform(train_df)
|
|
67
|
+
X_test = pipe.transform(test_df)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
income:
|
|
72
|
+
+ outliers_report - skew 3.22 is moderate (>= 1.0); IQR fence widened to k=3.0 for the asymmetry
|
|
73
|
+
+ impute_median - 2.0% missing; median rather than mean because it is unaffected by
|
|
74
|
+
the skew (3.22) and by outliers
|
|
75
|
+
+ transform_log1p - skew 3.22 is moderate and the column is non-negative (min 2576.27),
|
|
76
|
+
so log1p applies and is invertible
|
|
77
|
+
+ scale_robust - skew 3.22; robust scaling (median and IQR) rather than standard,
|
|
78
|
+
whose standard deviation is dominated by the tail
|
|
79
|
+
|
|
80
|
+
city:
|
|
81
|
+
+ group_rare_categories - 163 levels; those appearing in fewer than 5 rows (1.0%) are
|
|
82
|
+
grouped, since they cannot support a reliable estimate
|
|
83
|
+
+ encode_target - 163 levels exceeds the 50-level one-hot ceiling; target encoding is
|
|
84
|
+
used with 5-fold cross-fitting so no row is encoded using its own target
|
|
85
|
+
|
|
86
|
+
customer_id:
|
|
87
|
+
x dropped - identifier: 100.0% of values are distinct, so it cannot generalise beyond
|
|
88
|
+
the rows it was fitted on
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## Why it exists
|
|
94
|
+
|
|
95
|
+
It was built by mining common notebook workflows for the EDA and
|
|
96
|
+
preprocessing workflow they have in common — a broad survey of notebook workflows.
|
|
97
|
+
The findings are written up in [`docs/design-rationale.md`](docs/design-rationale.md), and
|
|
98
|
+
they shaped every design decision:
|
|
99
|
+
|
|
100
|
+
- The dtype-based column split (`select_dtypes(include=['int64','float64'])`) appears
|
|
101
|
+
**39 times** and is the largest single source of error: it sends a zip code and a
|
|
102
|
+
temperature down the same path. `edaprep` infers a *semantic* type and reports its
|
|
103
|
+
confidence.
|
|
104
|
+
- The IQR outlier fence is rewritten **12 times**, the z-score fence 6 times, with the
|
|
105
|
+
multiplier drifting between 1.5 and 3.0 for no recorded reason. Both are now single
|
|
106
|
+
parameterised, named, reported operations.
|
|
107
|
+
- **Leakage is easy to introduce.** One fits a `StandardScaler` on the full
|
|
108
|
+
frame, writes the result to CSV, and splits afterwards. `edaprep` makes that
|
|
109
|
+
structurally impossible rather than merely discouraged.
|
|
110
|
+
- Two notebooks independently maintain *parallel preprocessing branches* for tree and
|
|
111
|
+
linear models. That insight became `model_family`, a first-class planning input.
|
|
112
|
+
|
|
113
|
+
## Installation
|
|
114
|
+
|
|
115
|
+
Not yet published to PyPI. Install from the tagged release:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
pip install "git+https://github.com/bijay-odyssey/edaprep@v0.1.0"
|
|
119
|
+
pip install "edaprep[visualization] @ git+https://github.com/bijay-odyssey/edaprep@v0.1.0"
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Once it is on PyPI, this becomes:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pip install edaprep # core: numpy, pandas, scipy
|
|
126
|
+
pip install "edaprep[visualization]" # + matplotlib
|
|
127
|
+
pip install "edaprep[advanced]" # + scikit-learn
|
|
128
|
+
pip install "edaprep[all]"
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Python 3.9–3.13, tested on Linux, macOS and Windows.
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## What it does
|
|
136
|
+
|
|
137
|
+
### Understand a dataset
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
profile = edaprep.profile(df, target="churn")
|
|
141
|
+
print(profile.summary())
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
```
|
|
145
|
+
Dataset
|
|
146
|
+
600 rows x 18 columns
|
|
147
|
+
300.3 KB in memory
|
|
148
|
+
628 missing cells (5.81%)
|
|
149
|
+
target: churn (classification, 2 classes, minority/majority ratio 0.232)
|
|
150
|
+
|
|
151
|
+
Semantic types
|
|
152
|
+
numeric 5
|
|
153
|
+
binary 4
|
|
154
|
+
categorical 3
|
|
155
|
+
...
|
|
156
|
+
|
|
157
|
+
Data-quality findings
|
|
158
|
+
[x] 1 column(s) are almost perfectly associated with the target (>= 0.98). This
|
|
159
|
+
usually means the column encodes the answer: 'leaky'.
|
|
160
|
+
[!] 2 column(s) contain placeholder strings that most likely mean 'missing' but are
|
|
161
|
+
not recognised as NaN: 'workclass', 'occupation'.
|
|
162
|
+
[!] 1 group(s) of identical columns: income=income_copy
|
|
163
|
+
[i] 1 column pair(s) go missing together, which usually means a shared cause:
|
|
164
|
+
income~income_copy (1.00)
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
### Explore it
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
report = edaprep.EDA(df, target="churn").analyze("standard")
|
|
171
|
+
print(report.summary())
|
|
172
|
+
report.numerical # a DataFrame
|
|
173
|
+
report.to_html("eda.html")
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Three levels that differ in work done, not just in what is shown: `quick` skips every
|
|
177
|
+
O(n log n) and O(p²) computation, `standard` adds moments, outliers, correlation and
|
|
178
|
+
target relationships, `deep` adds VIF and significance tests with a
|
|
179
|
+
Benjamini-Hochberg adjustment.
|
|
180
|
+
|
|
181
|
+
### Prepare it
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
pipe = edaprep.AutoPipeline(target="churn", model_family="linear", random_state=42)
|
|
185
|
+
X_train = pipe.fit_transform(train_df)
|
|
186
|
+
X_test = pipe.transform(test_df)
|
|
187
|
+
|
|
188
|
+
pipe.plan_ # the decisions, serialisable and editable
|
|
189
|
+
pipe.report_ # what actually happened, with counts
|
|
190
|
+
pipe.transformations_ # one row per decision, as a DataFrame
|
|
191
|
+
pipe.statistics_ # every learned parameter
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Or say exactly what should happen:
|
|
195
|
+
|
|
196
|
+
```python
|
|
197
|
+
pipe = (
|
|
198
|
+
edaprep.Pipeline(target="churn")
|
|
199
|
+
.flag_missing()
|
|
200
|
+
.handle_outliers(strategy="clip")
|
|
201
|
+
.handle_missing()
|
|
202
|
+
.encode_categorical()
|
|
203
|
+
.scale_numeric()
|
|
204
|
+
)
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
### Override anything
|
|
208
|
+
|
|
209
|
+
```python
|
|
210
|
+
config = edaprep.Config(random_state=42)
|
|
211
|
+
config.column("age").imputation = "mean"
|
|
212
|
+
config.column("income").outlier_strategy = "clip"
|
|
213
|
+
config.column("city").encoding = "frequency"
|
|
214
|
+
config.column("zip").semantic_type = "categorical"
|
|
215
|
+
config.thresholds.skew_heavy = 4.0
|
|
216
|
+
|
|
217
|
+
pipe = edaprep.AutoPipeline(target="churn", config=config)
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
Overrides are tagged in the plan, so `explain()` marks them as yours rather than
|
|
221
|
+
presenting them as the planner's reasoning.
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
## Design guarantees
|
|
226
|
+
|
|
227
|
+
**No leakage, structurally.** Learned state lives only in attributes written inside
|
|
228
|
+
`fit`; `transform` is a pure function of that state. The property is asserted directly:
|
|
229
|
+
a test transforms a frame whole and then row by row and requires identical output, which
|
|
230
|
+
fails immediately if anything recomputes a statistic at transform time.
|
|
231
|
+
|
|
232
|
+
**Nothing silent.** Dropped columns, imputed values, grouped categories, clipped rows
|
|
233
|
+
and unseen categories are all counted and reported. `edaprep` never calls
|
|
234
|
+
`warnings.filterwarnings`.
|
|
235
|
+
|
|
236
|
+
**Everything explainable.** Every automatic decision carries an English rationale naming
|
|
237
|
+
the measurement behind it. The plan is inert, serialisable data — printable, diffable,
|
|
238
|
+
storable next to a model artefact, and re-executable.
|
|
239
|
+
|
|
240
|
+
**Reproducible.** `random_state` seeds every stochastic step. The report records the
|
|
241
|
+
library version, the configuration, the seed, whether profiling sampled, and every
|
|
242
|
+
learned parameter.
|
|
243
|
+
|
|
244
|
+
**Conservative.** Outliers are reported, not deleted, by default. Duplicate rows are
|
|
245
|
+
reported, not removed — repeated observations are legitimate in transactional data.
|
|
246
|
+
Class imbalance is measured and reported; resampling is a modelling decision that
|
|
247
|
+
belongs after the split, so `edaprep` does not do it.
|
|
248
|
+
|
|
249
|
+
---
|
|
250
|
+
|
|
251
|
+
## Performance
|
|
252
|
+
|
|
253
|
+
Measured, not asserted. See [`docs/performance.md`](docs/performance.md).
|
|
254
|
+
|
|
255
|
+
| operation | edaprep | baseline |
|
|
256
|
+
|---|---|---|
|
|
257
|
+
| `Scaler` (standard) | 6.9 ms | sklearn `StandardScaler` 26.7 ms |
|
|
258
|
+
| `MissingValueHandler` (median) | 5.2 ms | sklearn `SimpleImputer` 17.0 ms |
|
|
259
|
+
| `OutlierHandler` (IQR clip) | 25.2 ms | the usual IQR block 41.4 ms |
|
|
260
|
+
| `numeric_block_stats` (20k × 300) | 578 ms | equivalent pandas loop 1056 ms |
|
|
261
|
+
| `AutoPipeline.transform` | 240 ms / 35.6 MiB | `ColumnTransformer` 104 ms / 67.2 MiB |
|
|
262
|
+
|
|
263
|
+
100,000 rows unless stated. The most instructive result is one that went the other way:
|
|
264
|
+
a hand-written NumPy kernel in this library turned out to be **2.1× slower** than the
|
|
265
|
+
pandas code it replaced, so it was deleted. That story is in `docs/performance.md` §1.
|
|
266
|
+
|
|
267
|
+
No native code. Nothing here is un-vectorisable, and the one place a hand-written kernel
|
|
268
|
+
looked promising was slower than pandas.
|
|
269
|
+
|
|
270
|
+
---
|
|
271
|
+
|
|
272
|
+
## Documentation
|
|
273
|
+
|
|
274
|
+
| | |
|
|
275
|
+
|---|---|
|
|
276
|
+
| [Workflow mining](docs/design-rationale.md) | what 13 repositories revealed, and the 9 defects found |
|
|
277
|
+
| [Architecture](docs/architecture.md) | package design, the planner, execution model |
|
|
278
|
+
| [User guide](docs/guide.md) | installation to production, with the train/test workflow |
|
|
279
|
+
| [Performance](docs/performance.md) | benchmarks, method, and what optimisation actually changed |
|
|
280
|
+
| [Extending](docs/extending.md) | custom transformers, rules and backends |
|
|
281
|
+
| [Example](examples/end_to_end.py) | raw dataset to ML-ready, end to end |
|
|
282
|
+
|
|
283
|
+
---
|
|
284
|
+
|
|
285
|
+
## Scope
|
|
286
|
+
|
|
287
|
+
**In:** dataset inspection, EDA, data quality, cleaning, missing values, duplicates,
|
|
288
|
+
outliers, dtype inference, categorical encoding, numeric transformation, scaling,
|
|
289
|
+
feature selection, datetime expansion, leakage-safe train/test preparation, pipelines,
|
|
290
|
+
reporting.
|
|
291
|
+
|
|
292
|
+
**Out, deliberately:** model training, resampling, hyperparameter search, NLP,
|
|
293
|
+
forecasting, deep learning, distributed execution. Extension points exist for each
|
|
294
|
+
(`docs/extending.md`), and none is implemented in v1.
|
|
295
|
+
|
|
296
|
+
## Development
|
|
297
|
+
|
|
298
|
+
```bash
|
|
299
|
+
pip install -e ".[dev]"
|
|
300
|
+
pytest # 353 tests
|
|
301
|
+
python benchmarks/bench.py
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
## Licence
|
|
305
|
+
|
|
306
|
+
MIT.
|
edaprep-0.1.0/README.md
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# edaprep
|
|
2
|
+
|
|
3
|
+
[](https://github.com/bijay-odyssey/edaprep/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/edaprep/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Transparent, leakage-safe EDA and ML preprocessing, with an explainable planner.
|
|
8
|
+
|
|
9
|
+
`edaprep` looks at a dataset, works out which preprocessing operations actually apply
|
|
10
|
+
to it, tells you what it intends to do and why, and then does it — fitting every
|
|
11
|
+
statistic on the training data alone.
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
import edaprep
|
|
15
|
+
|
|
16
|
+
pipe = edaprep.AutoPipeline(target="churn", model_family="tree", random_state=42)
|
|
17
|
+
pipe.fit(train_df)
|
|
18
|
+
pipe.explain()
|
|
19
|
+
|
|
20
|
+
X_train = pipe.transform(train_df)
|
|
21
|
+
X_test = pipe.transform(test_df)
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
income:
|
|
26
|
+
+ outliers_report - skew 3.22 is moderate (>= 1.0); IQR fence widened to k=3.0 for the asymmetry
|
|
27
|
+
+ impute_median - 2.0% missing; median rather than mean because it is unaffected by
|
|
28
|
+
the skew (3.22) and by outliers
|
|
29
|
+
+ transform_log1p - skew 3.22 is moderate and the column is non-negative (min 2576.27),
|
|
30
|
+
so log1p applies and is invertible
|
|
31
|
+
+ scale_robust - skew 3.22; robust scaling (median and IQR) rather than standard,
|
|
32
|
+
whose standard deviation is dominated by the tail
|
|
33
|
+
|
|
34
|
+
city:
|
|
35
|
+
+ group_rare_categories - 163 levels; those appearing in fewer than 5 rows (1.0%) are
|
|
36
|
+
grouped, since they cannot support a reliable estimate
|
|
37
|
+
+ encode_target - 163 levels exceeds the 50-level one-hot ceiling; target encoding is
|
|
38
|
+
used with 5-fold cross-fitting so no row is encoded using its own target
|
|
39
|
+
|
|
40
|
+
customer_id:
|
|
41
|
+
x dropped - identifier: 100.0% of values are distinct, so it cannot generalise beyond
|
|
42
|
+
the rows it was fitted on
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## Why it exists
|
|
48
|
+
|
|
49
|
+
It was built by mining common notebook workflows for the EDA and
|
|
50
|
+
preprocessing workflow they have in common — a broad survey of notebook workflows.
|
|
51
|
+
The findings are written up in [`docs/design-rationale.md`](docs/design-rationale.md), and
|
|
52
|
+
they shaped every design decision:
|
|
53
|
+
|
|
54
|
+
- The dtype-based column split (`select_dtypes(include=['int64','float64'])`) appears
|
|
55
|
+
**39 times** and is the largest single source of error: it sends a zip code and a
|
|
56
|
+
temperature down the same path. `edaprep` infers a *semantic* type and reports its
|
|
57
|
+
confidence.
|
|
58
|
+
- The IQR outlier fence is rewritten **12 times**, the z-score fence 6 times, with the
|
|
59
|
+
multiplier drifting between 1.5 and 3.0 for no recorded reason. Both are now single
|
|
60
|
+
parameterised, named, reported operations.
|
|
61
|
+
- **Leakage is easy to introduce.** One fits a `StandardScaler` on the full
|
|
62
|
+
frame, writes the result to CSV, and splits afterwards. `edaprep` makes that
|
|
63
|
+
structurally impossible rather than merely discouraged.
|
|
64
|
+
- Two notebooks independently maintain *parallel preprocessing branches* for tree and
|
|
65
|
+
linear models. That insight became `model_family`, a first-class planning input.
|
|
66
|
+
|
|
67
|
+
## Installation
|
|
68
|
+
|
|
69
|
+
Not yet published to PyPI. Install from the tagged release:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pip install "git+https://github.com/bijay-odyssey/edaprep@v0.1.0"
|
|
73
|
+
pip install "edaprep[visualization] @ git+https://github.com/bijay-odyssey/edaprep@v0.1.0"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Once it is on PyPI, this becomes:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
pip install edaprep # core: numpy, pandas, scipy
|
|
80
|
+
pip install "edaprep[visualization]" # + matplotlib
|
|
81
|
+
pip install "edaprep[advanced]" # + scikit-learn
|
|
82
|
+
pip install "edaprep[all]"
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Python 3.9–3.13, tested on Linux, macOS and Windows.
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## What it does
|
|
90
|
+
|
|
91
|
+
### Understand a dataset
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
profile = edaprep.profile(df, target="churn")
|
|
95
|
+
print(profile.summary())
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
```
|
|
99
|
+
Dataset
|
|
100
|
+
600 rows x 18 columns
|
|
101
|
+
300.3 KB in memory
|
|
102
|
+
628 missing cells (5.81%)
|
|
103
|
+
target: churn (classification, 2 classes, minority/majority ratio 0.232)
|
|
104
|
+
|
|
105
|
+
Semantic types
|
|
106
|
+
numeric 5
|
|
107
|
+
binary 4
|
|
108
|
+
categorical 3
|
|
109
|
+
...
|
|
110
|
+
|
|
111
|
+
Data-quality findings
|
|
112
|
+
[x] 1 column(s) are almost perfectly associated with the target (>= 0.98). This
|
|
113
|
+
usually means the column encodes the answer: 'leaky'.
|
|
114
|
+
[!] 2 column(s) contain placeholder strings that most likely mean 'missing' but are
|
|
115
|
+
not recognised as NaN: 'workclass', 'occupation'.
|
|
116
|
+
[!] 1 group(s) of identical columns: income=income_copy
|
|
117
|
+
[i] 1 column pair(s) go missing together, which usually means a shared cause:
|
|
118
|
+
income~income_copy (1.00)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### Explore it
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
report = edaprep.EDA(df, target="churn").analyze("standard")
|
|
125
|
+
print(report.summary())
|
|
126
|
+
report.numerical # a DataFrame
|
|
127
|
+
report.to_html("eda.html")
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Three levels that differ in work done, not just in what is shown: `quick` skips every
|
|
131
|
+
O(n log n) and O(p²) computation, `standard` adds moments, outliers, correlation and
|
|
132
|
+
target relationships, `deep` adds VIF and significance tests with a
|
|
133
|
+
Benjamini-Hochberg adjustment.
|
|
134
|
+
|
|
135
|
+
### Prepare it
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
pipe = edaprep.AutoPipeline(target="churn", model_family="linear", random_state=42)
|
|
139
|
+
X_train = pipe.fit_transform(train_df)
|
|
140
|
+
X_test = pipe.transform(test_df)
|
|
141
|
+
|
|
142
|
+
pipe.plan_ # the decisions, serialisable and editable
|
|
143
|
+
pipe.report_ # what actually happened, with counts
|
|
144
|
+
pipe.transformations_ # one row per decision, as a DataFrame
|
|
145
|
+
pipe.statistics_ # every learned parameter
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Or say exactly what should happen:
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
pipe = (
|
|
152
|
+
edaprep.Pipeline(target="churn")
|
|
153
|
+
.flag_missing()
|
|
154
|
+
.handle_outliers(strategy="clip")
|
|
155
|
+
.handle_missing()
|
|
156
|
+
.encode_categorical()
|
|
157
|
+
.scale_numeric()
|
|
158
|
+
)
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### Override anything
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
config = edaprep.Config(random_state=42)
|
|
165
|
+
config.column("age").imputation = "mean"
|
|
166
|
+
config.column("income").outlier_strategy = "clip"
|
|
167
|
+
config.column("city").encoding = "frequency"
|
|
168
|
+
config.column("zip").semantic_type = "categorical"
|
|
169
|
+
config.thresholds.skew_heavy = 4.0
|
|
170
|
+
|
|
171
|
+
pipe = edaprep.AutoPipeline(target="churn", config=config)
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Overrides are tagged in the plan, so `explain()` marks them as yours rather than
|
|
175
|
+
presenting them as the planner's reasoning.
|
|
176
|
+
|
|
177
|
+
---
|
|
178
|
+
|
|
179
|
+
## Design guarantees
|
|
180
|
+
|
|
181
|
+
**No leakage, structurally.** Learned state lives only in attributes written inside
|
|
182
|
+
`fit`; `transform` is a pure function of that state. The property is asserted directly:
|
|
183
|
+
a test transforms a frame whole and then row by row and requires identical output, which
|
|
184
|
+
fails immediately if anything recomputes a statistic at transform time.
|
|
185
|
+
|
|
186
|
+
**Nothing silent.** Dropped columns, imputed values, grouped categories, clipped rows
|
|
187
|
+
and unseen categories are all counted and reported. `edaprep` never calls
|
|
188
|
+
`warnings.filterwarnings`.
|
|
189
|
+
|
|
190
|
+
**Everything explainable.** Every automatic decision carries an English rationale naming
|
|
191
|
+
the measurement behind it. The plan is inert, serialisable data — printable, diffable,
|
|
192
|
+
storable next to a model artefact, and re-executable.
|
|
193
|
+
|
|
194
|
+
**Reproducible.** `random_state` seeds every stochastic step. The report records the
|
|
195
|
+
library version, the configuration, the seed, whether profiling sampled, and every
|
|
196
|
+
learned parameter.
|
|
197
|
+
|
|
198
|
+
**Conservative.** Outliers are reported, not deleted, by default. Duplicate rows are
|
|
199
|
+
reported, not removed — repeated observations are legitimate in transactional data.
|
|
200
|
+
Class imbalance is measured and reported; resampling is a modelling decision that
|
|
201
|
+
belongs after the split, so `edaprep` does not do it.
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## Performance
|
|
206
|
+
|
|
207
|
+
Measured, not asserted. See [`docs/performance.md`](docs/performance.md).
|
|
208
|
+
|
|
209
|
+
| operation | edaprep | baseline |
|
|
210
|
+
|---|---|---|
|
|
211
|
+
| `Scaler` (standard) | 6.9 ms | sklearn `StandardScaler` 26.7 ms |
|
|
212
|
+
| `MissingValueHandler` (median) | 5.2 ms | sklearn `SimpleImputer` 17.0 ms |
|
|
213
|
+
| `OutlierHandler` (IQR clip) | 25.2 ms | the usual IQR block 41.4 ms |
|
|
214
|
+
| `numeric_block_stats` (20k × 300) | 578 ms | equivalent pandas loop 1056 ms |
|
|
215
|
+
| `AutoPipeline.transform` | 240 ms / 35.6 MiB | `ColumnTransformer` 104 ms / 67.2 MiB |
|
|
216
|
+
|
|
217
|
+
100,000 rows unless stated. The most instructive result is one that went the other way:
|
|
218
|
+
a hand-written NumPy kernel in this library turned out to be **2.1× slower** than the
|
|
219
|
+
pandas code it replaced, so it was deleted. That story is in `docs/performance.md` §1.
|
|
220
|
+
|
|
221
|
+
No native code. Nothing here is un-vectorisable, and the one place a hand-written kernel
|
|
222
|
+
looked promising was slower than pandas.
|
|
223
|
+
|
|
224
|
+
---
|
|
225
|
+
|
|
226
|
+
## Documentation
|
|
227
|
+
|
|
228
|
+
| | |
|
|
229
|
+
|---|---|
|
|
230
|
+
| [Workflow mining](docs/design-rationale.md) | what 13 repositories revealed, and the 9 defects found |
|
|
231
|
+
| [Architecture](docs/architecture.md) | package design, the planner, execution model |
|
|
232
|
+
| [User guide](docs/guide.md) | installation to production, with the train/test workflow |
|
|
233
|
+
| [Performance](docs/performance.md) | benchmarks, method, and what optimisation actually changed |
|
|
234
|
+
| [Extending](docs/extending.md) | custom transformers, rules and backends |
|
|
235
|
+
| [Example](examples/end_to_end.py) | raw dataset to ML-ready, end to end |
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
## Scope
|
|
240
|
+
|
|
241
|
+
**In:** dataset inspection, EDA, data quality, cleaning, missing values, duplicates,
|
|
242
|
+
outliers, dtype inference, categorical encoding, numeric transformation, scaling,
|
|
243
|
+
feature selection, datetime expansion, leakage-safe train/test preparation, pipelines,
|
|
244
|
+
reporting.
|
|
245
|
+
|
|
246
|
+
**Out, deliberately:** model training, resampling, hyperparameter search, NLP,
|
|
247
|
+
forecasting, deep learning, distributed execution. Extension points exist for each
|
|
248
|
+
(`docs/extending.md`), and none is implemented in v1.
|
|
249
|
+
|
|
250
|
+
## Development
|
|
251
|
+
|
|
252
|
+
```bash
|
|
253
|
+
pip install -e ".[dev]"
|
|
254
|
+
pytest # 353 tests
|
|
255
|
+
python benchmarks/bench.py
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
## Licence
|
|
259
|
+
|
|
260
|
+
MIT.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "edaprep"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Transparent, leakage-safe EDA and ML preprocessing with an explainable planner."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "bijay", email = "bijaybeezoe@gmail.com" }]
|
|
13
|
+
keywords = ["eda", "preprocessing", "data-cleaning", "machine-learning", "pandas"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Science/Research",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.9",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Topic :: Scientific/Engineering",
|
|
25
|
+
"Typing :: Typed",
|
|
26
|
+
]
|
|
27
|
+
dependencies = [
|
|
28
|
+
"numpy>=1.21",
|
|
29
|
+
"pandas>=1.5",
|
|
30
|
+
"scipy>=1.7",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
visualization = ["matplotlib>=3.5"]
|
|
35
|
+
advanced = ["scikit-learn>=1.1"]
|
|
36
|
+
arrow = ["pyarrow>=10.0"]
|
|
37
|
+
all = ["matplotlib>=3.5", "scikit-learn>=1.1", "pyarrow>=10.0"]
|
|
38
|
+
dev = [
|
|
39
|
+
"pytest>=7.0",
|
|
40
|
+
"pytest-cov>=4.0",
|
|
41
|
+
"hypothesis>=6.60",
|
|
42
|
+
"scikit-learn>=1.1",
|
|
43
|
+
"matplotlib>=3.5",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
[project.urls]
|
|
47
|
+
Documentation = "https://github.com/bijay-odyssey/edaprep#readme"
|
|
48
|
+
Source = "https://github.com/bijay-odyssey/edaprep"
|
|
49
|
+
Changelog = "https://github.com/bijay-odyssey/edaprep/blob/main/CHANGELOG.md"
|
|
50
|
+
Issues = "https://github.com/bijay-odyssey/edaprep/issues"
|
|
51
|
+
|
|
52
|
+
[tool.setuptools.dynamic]
|
|
53
|
+
version = { attr = "edaprep._version.__version__" }
|
|
54
|
+
|
|
55
|
+
[tool.setuptools.packages.find]
|
|
56
|
+
where = ["src"]
|
|
57
|
+
|
|
58
|
+
[tool.setuptools.package-data]
|
|
59
|
+
edaprep = ["py.typed"]
|
|
60
|
+
|
|
61
|
+
[tool.pytest.ini_options]
|
|
62
|
+
testpaths = ["tests"]
|
|
63
|
+
addopts = "-q --strict-markers"
|
|
64
|
+
markers = [
|
|
65
|
+
"slow: long-running tests (deselect with '-m \"not slow\"')",
|
|
66
|
+
"benchmark: performance measurement, not correctness",
|
|
67
|
+
]
|
|
68
|
+
filterwarnings = ["error::RuntimeWarning"]
|
|
69
|
+
|
|
70
|
+
[tool.ruff]
|
|
71
|
+
line-length = 96
|
|
72
|
+
target-version = "py39"
|
|
73
|
+
|
|
74
|
+
[tool.ruff.lint]
|
|
75
|
+
select = ["E", "F", "W", "I", "UP", "B", "C4", "SIM", "RET"]
|
|
76
|
+
ignore = [
|
|
77
|
+
"B008",
|
|
78
|
+
# PEP 585/604 modernisation (dict[str, int], X | None). Deliberately not adopted:
|
|
79
|
+
# the package supports Python 3.9, where those forms are only legal inside
|
|
80
|
+
# annotations (via `from __future__ import annotations`) and not in the runtime
|
|
81
|
+
# positions this codebase also uses them in -- dataclass field types resolved by
|
|
82
|
+
# `dataclasses.fields`, and `Stage.coerce` argument checks. Mixing the two
|
|
83
|
+
# spellings would be worse than consistently using `typing`.
|
|
84
|
+
"UP006",
|
|
85
|
+
"UP007",
|
|
86
|
+
"UP035",
|
|
87
|
+
"UP045",
|
|
88
|
+
# Quoted annotations are load-bearing here: `Axes` and `Figure` are imported only
|
|
89
|
+
# under TYPE_CHECKING so that matplotlib stays an optional dependency.
|
|
90
|
+
"UP037",
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
[tool.mypy]
|
|
94
|
+
python_version = "3.9"
|
|
95
|
+
warn_unused_configs = true
|
|
96
|
+
disallow_untyped_defs = false
|
|
97
|
+
ignore_missing_imports = true
|
edaprep-0.1.0/setup.cfg
ADDED