edaprep 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. edaprep-0.1.0/LICENSE +21 -0
  2. edaprep-0.1.0/PKG-INFO +306 -0
  3. edaprep-0.1.0/README.md +260 -0
  4. edaprep-0.1.0/pyproject.toml +97 -0
  5. edaprep-0.1.0/setup.cfg +4 -0
  6. edaprep-0.1.0/src/edaprep/__init__.py +188 -0
  7. edaprep-0.1.0/src/edaprep/_version.py +1 -0
  8. edaprep-0.1.0/src/edaprep/backends/__init__.py +11 -0
  9. edaprep-0.1.0/src/edaprep/backends/base.py +167 -0
  10. edaprep-0.1.0/src/edaprep/backends/pandas_backend.py +128 -0
  11. edaprep-0.1.0/src/edaprep/config.py +504 -0
  12. edaprep-0.1.0/src/edaprep/core/__init__.py +7 -0
  13. edaprep-0.1.0/src/edaprep/core/base.py +314 -0
  14. edaprep-0.1.0/src/edaprep/core/context.py +81 -0
  15. edaprep-0.1.0/src/edaprep/core/journal.py +244 -0
  16. edaprep-0.1.0/src/edaprep/core/pipeline.py +587 -0
  17. edaprep-0.1.0/src/edaprep/eda/__init__.py +22 -0
  18. edaprep-0.1.0/src/edaprep/eda/analyzer.py +486 -0
  19. edaprep-0.1.0/src/edaprep/eda/categorical.py +100 -0
  20. edaprep-0.1.0/src/edaprep/eda/correlation.py +183 -0
  21. edaprep-0.1.0/src/edaprep/eda/numerical.py +86 -0
  22. edaprep-0.1.0/src/edaprep/eda/outliers.py +99 -0
  23. edaprep-0.1.0/src/edaprep/eda/target.py +214 -0
  24. edaprep-0.1.0/src/edaprep/exceptions.py +161 -0
  25. edaprep-0.1.0/src/edaprep/planning/__init__.py +16 -0
  26. edaprep-0.1.0/src/edaprep/planning/decisions.py +358 -0
  27. edaprep-0.1.0/src/edaprep/planning/planner.py +364 -0
  28. edaprep-0.1.0/src/edaprep/planning/rules.py +819 -0
  29. edaprep-0.1.0/src/edaprep/preprocessing/__init__.py +67 -0
  30. edaprep-0.1.0/src/edaprep/preprocessing/casting.py +260 -0
  31. edaprep-0.1.0/src/edaprep/preprocessing/datetime_features.py +208 -0
  32. edaprep-0.1.0/src/edaprep/preprocessing/duplicates.py +170 -0
  33. edaprep-0.1.0/src/edaprep/preprocessing/encoding.py +804 -0
  34. edaprep-0.1.0/src/edaprep/preprocessing/missing.py +363 -0
  35. edaprep-0.1.0/src/edaprep/preprocessing/outliers.py +569 -0
  36. edaprep-0.1.0/src/edaprep/preprocessing/scaling.py +228 -0
  37. edaprep-0.1.0/src/edaprep/preprocessing/selection.py +468 -0
  38. edaprep-0.1.0/src/edaprep/preprocessing/text.py +124 -0
  39. edaprep-0.1.0/src/edaprep/preprocessing/transformations.py +379 -0
  40. edaprep-0.1.0/src/edaprep/profiling/__init__.py +19 -0
  41. edaprep-0.1.0/src/edaprep/profiling/column_types.py +498 -0
  42. edaprep-0.1.0/src/edaprep/profiling/profiler.py +1095 -0
  43. edaprep-0.1.0/src/edaprep/profiling/quality.py +357 -0
  44. edaprep-0.1.0/src/edaprep/profiling/statistics.py +516 -0
  45. edaprep-0.1.0/src/edaprep/py.typed +0 -0
  46. edaprep-0.1.0/src/edaprep/reporting/__init__.py +5 -0
  47. edaprep-0.1.0/src/edaprep/reporting/html.py +173 -0
  48. edaprep-0.1.0/src/edaprep/reporting/report.py +317 -0
  49. edaprep-0.1.0/src/edaprep/types.py +189 -0
  50. edaprep-0.1.0/src/edaprep/visualization/__init__.py +33 -0
  51. edaprep-0.1.0/src/edaprep/visualization/plots.py +330 -0
  52. edaprep-0.1.0/src/edaprep.egg-info/PKG-INFO +306 -0
  53. edaprep-0.1.0/src/edaprep.egg-info/SOURCES.txt +61 -0
  54. edaprep-0.1.0/src/edaprep.egg-info/dependency_links.txt +1 -0
  55. edaprep-0.1.0/src/edaprep.egg-info/requires.txt +24 -0
  56. edaprep-0.1.0/src/edaprep.egg-info/top_level.txt +1 -0
  57. edaprep-0.1.0/tests/test_column_types.py +211 -0
  58. edaprep-0.1.0/tests/test_eda.py +420 -0
  59. edaprep-0.1.0/tests/test_leakage.py +385 -0
  60. edaprep-0.1.0/tests/test_pipeline.py +606 -0
  61. edaprep-0.1.0/tests/test_profiler.py +288 -0
  62. edaprep-0.1.0/tests/test_statistics.py +243 -0
  63. edaprep-0.1.0/tests/test_transformers.py +917 -0
edaprep-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 bijay
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
edaprep-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,306 @@
1
+ Metadata-Version: 2.4
2
+ Name: edaprep
3
+ Version: 0.1.0
4
+ Summary: Transparent, leakage-safe EDA and ML preprocessing with an explainable planner.
5
+ Author-email: bijay <bijaybeezoe@gmail.com>
6
+ License: MIT
7
+ Project-URL: Documentation, https://github.com/bijay-odyssey/edaprep#readme
8
+ Project-URL: Source, https://github.com/bijay-odyssey/edaprep
9
+ Project-URL: Changelog, https://github.com/bijay-odyssey/edaprep/blob/main/CHANGELOG.md
10
+ Project-URL: Issues, https://github.com/bijay-odyssey/edaprep/issues
11
+ Keywords: eda,preprocessing,data-cleaning,machine-learning,pandas
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Scientific/Engineering
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.9
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ Requires-Dist: numpy>=1.21
27
+ Requires-Dist: pandas>=1.5
28
+ Requires-Dist: scipy>=1.7
29
+ Provides-Extra: visualization
30
+ Requires-Dist: matplotlib>=3.5; extra == "visualization"
31
+ Provides-Extra: advanced
32
+ Requires-Dist: scikit-learn>=1.1; extra == "advanced"
33
+ Provides-Extra: arrow
34
+ Requires-Dist: pyarrow>=10.0; extra == "arrow"
35
+ Provides-Extra: all
36
+ Requires-Dist: matplotlib>=3.5; extra == "all"
37
+ Requires-Dist: scikit-learn>=1.1; extra == "all"
38
+ Requires-Dist: pyarrow>=10.0; extra == "all"
39
+ Provides-Extra: dev
40
+ Requires-Dist: pytest>=7.0; extra == "dev"
41
+ Requires-Dist: pytest-cov>=4.0; extra == "dev"
42
+ Requires-Dist: hypothesis>=6.60; extra == "dev"
43
+ Requires-Dist: scikit-learn>=1.1; extra == "dev"
44
+ Requires-Dist: matplotlib>=3.5; extra == "dev"
45
+ Dynamic: license-file
46
+
47
+ # edaprep
48
+
49
+ [![CI](https://github.com/bijay-odyssey/edaprep/actions/workflows/ci.yml/badge.svg)](https://github.com/bijay-odyssey/edaprep/actions/workflows/ci.yml)
50
+ [![Python](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)](https://pypi.org/project/edaprep/)
51
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
52
+
53
+ Transparent, leakage-safe EDA and ML preprocessing, with an explainable planner.
54
+
55
+ `edaprep` looks at a dataset, works out which preprocessing operations actually apply
56
+ to it, tells you what it intends to do and why, and then does it — fitting every
57
+ statistic on the training data alone.
58
+
59
+ ```python
60
+ import edaprep
61
+
62
+ pipe = edaprep.AutoPipeline(target="churn", model_family="tree", random_state=42)
63
+ pipe.fit(train_df)
64
+ pipe.explain()
65
+
66
+ X_train = pipe.transform(train_df)
67
+ X_test = pipe.transform(test_df)
68
+ ```
69
+
70
+ ```
71
+ income:
72
+ + outliers_report - skew 3.22 is moderate (>= 1.0); IQR fence widened to k=3.0 for the asymmetry
73
+ + impute_median - 2.0% missing; median rather than mean because it is unaffected by
74
+ the skew (3.22) and by outliers
75
+ + transform_log1p - skew 3.22 is moderate and the column is non-negative (min 2576.27),
76
+ so log1p applies and is invertible
77
+ + scale_robust - skew 3.22; robust scaling (median and IQR) rather than standard,
78
+ whose standard deviation is dominated by the tail
79
+
80
+ city:
81
+ + group_rare_categories - 163 levels; those appearing in fewer than 5 rows (1.0%) are
82
+ grouped, since they cannot support a reliable estimate
83
+ + encode_target - 163 levels exceeds the 50-level one-hot ceiling; target encoding is
84
+ used with 5-fold cross-fitting so no row is encoded using its own target
85
+
86
+ customer_id:
87
+ x dropped - identifier: 100.0% of values are distinct, so it cannot generalise beyond
88
+ the rows it was fitted on
89
+ ```
90
+
91
+ ---
92
+
93
+ ## Why it exists
94
+
95
+ It was built by mining common notebook workflows for the EDA and
96
+ preprocessing workflow they have in common — a broad survey of notebook workflows.
97
+ The findings are written up in [`docs/design-rationale.md`](docs/design-rationale.md), and
98
+ they shaped every design decision:
99
+
100
+ - The dtype-based column split (`select_dtypes(include=['int64','float64'])`) appears
101
+ **39 times** and is the largest single source of error: it sends a zip code and a
102
+ temperature down the same path. `edaprep` infers a *semantic* type and reports its
103
+ confidence.
104
+ - The IQR outlier fence is rewritten **12 times**, the z-score fence 6 times, with the
105
+ multiplier drifting between 1.5 and 3.0 for no recorded reason. Both are now single
106
+ parameterised, named, reported operations.
107
+ - **Leakage is easy to introduce.** One fits a `StandardScaler` on the full
108
+ frame, writes the result to CSV, and splits afterwards. `edaprep` makes that
109
+ structurally impossible rather than merely discouraged.
110
+ - Two notebooks independently maintain *parallel preprocessing branches* for tree and
111
+ linear models. That insight became `model_family`, a first-class planning input.
112
+
113
+ ## Installation
114
+
115
+ Not yet published to PyPI. Install from the tagged release:
116
+
117
+ ```bash
118
+ pip install "git+https://github.com/bijay-odyssey/edaprep@v0.1.0"
119
+ pip install "edaprep[visualization] @ git+https://github.com/bijay-odyssey/edaprep@v0.1.0"
120
+ ```
121
+
122
+ Once it is on PyPI, this becomes:
123
+
124
+ ```bash
125
+ pip install edaprep # core: numpy, pandas, scipy
126
+ pip install "edaprep[visualization]" # + matplotlib
127
+ pip install "edaprep[advanced]" # + scikit-learn
128
+ pip install "edaprep[all]"
129
+ ```
130
+
131
+ Python 3.9–3.13, tested on Linux, macOS and Windows.
132
+
133
+ ---
134
+
135
+ ## What it does
136
+
137
+ ### Understand a dataset
138
+
139
+ ```python
140
+ profile = edaprep.profile(df, target="churn")
141
+ print(profile.summary())
142
+ ```
143
+
144
+ ```
145
+ Dataset
146
+ 600 rows x 18 columns
147
+ 300.3 KB in memory
148
+ 628 missing cells (5.81%)
149
+ target: churn (classification, 2 classes, minority/majority ratio 0.232)
150
+
151
+ Semantic types
152
+ numeric 5
153
+ binary 4
154
+ categorical 3
155
+ ...
156
+
157
+ Data-quality findings
158
+ [x] 1 column(s) are almost perfectly associated with the target (>= 0.98). This
159
+ usually means the column encodes the answer: 'leaky'.
160
+ [!] 2 column(s) contain placeholder strings that most likely mean 'missing' but are
161
+ not recognised as NaN: 'workclass', 'occupation'.
162
+ [!] 1 group(s) of identical columns: income=income_copy
163
+ [i] 1 column pair(s) go missing together, which usually means a shared cause:
164
+ income~income_copy (1.00)
165
+ ```
166
+
167
+ ### Explore it
168
+
169
+ ```python
170
+ report = edaprep.EDA(df, target="churn").analyze("standard")
171
+ print(report.summary())
172
+ report.numerical # a DataFrame
173
+ report.to_html("eda.html")
174
+ ```
175
+
176
+ Three levels that differ in work done, not just in what is shown: `quick` skips every
177
+ O(n log n) and O(p²) computation, `standard` adds moments, outliers, correlation and
178
+ target relationships, `deep` adds VIF and significance tests with a
179
+ Benjamini-Hochberg adjustment.
180
+
181
+ ### Prepare it
182
+
183
+ ```python
184
+ pipe = edaprep.AutoPipeline(target="churn", model_family="linear", random_state=42)
185
+ X_train = pipe.fit_transform(train_df)
186
+ X_test = pipe.transform(test_df)
187
+
188
+ pipe.plan_ # the decisions, serialisable and editable
189
+ pipe.report_ # what actually happened, with counts
190
+ pipe.transformations_ # one row per decision, as a DataFrame
191
+ pipe.statistics_ # every learned parameter
192
+ ```
193
+
194
+ Or say exactly what should happen:
195
+
196
+ ```python
197
+ pipe = (
198
+ edaprep.Pipeline(target="churn")
199
+ .flag_missing()
200
+ .handle_outliers(strategy="clip")
201
+ .handle_missing()
202
+ .encode_categorical()
203
+ .scale_numeric()
204
+ )
205
+ ```
206
+
207
+ ### Override anything
208
+
209
+ ```python
210
+ config = edaprep.Config(random_state=42)
211
+ config.column("age").imputation = "mean"
212
+ config.column("income").outlier_strategy = "clip"
213
+ config.column("city").encoding = "frequency"
214
+ config.column("zip").semantic_type = "categorical"
215
+ config.thresholds.skew_heavy = 4.0
216
+
217
+ pipe = edaprep.AutoPipeline(target="churn", config=config)
218
+ ```
219
+
220
+ Overrides are tagged in the plan, so `explain()` marks them as yours rather than
221
+ presenting them as the planner's reasoning.
222
+
223
+ ---
224
+
225
+ ## Design guarantees
226
+
227
+ **No leakage, structurally.** Learned state lives only in attributes written inside
228
+ `fit`; `transform` is a pure function of that state. The property is asserted directly:
229
+ a test transforms a frame whole and then row by row and requires identical output, which
230
+ fails immediately if anything recomputes a statistic at transform time.
231
+
232
+ **Nothing silent.** Dropped columns, imputed values, grouped categories, clipped rows
233
+ and unseen categories are all counted and reported. `edaprep` never calls
234
+ `warnings.filterwarnings`.
235
+
236
+ **Everything explainable.** Every automatic decision carries an English rationale naming
237
+ the measurement behind it. The plan is inert, serialisable data — printable, diffable,
238
+ storable next to a model artefact, and re-executable.
239
+
240
+ **Reproducible.** `random_state` seeds every stochastic step. The report records the
241
+ library version, the configuration, the seed, whether profiling sampled, and every
242
+ learned parameter.
243
+
244
+ **Conservative.** Outliers are reported, not deleted, by default. Duplicate rows are
245
+ reported, not removed — repeated observations are legitimate in transactional data.
246
+ Class imbalance is measured and reported; resampling is a modelling decision that
247
+ belongs after the split, so `edaprep` does not do it.
248
+
249
+ ---
250
+
251
+ ## Performance
252
+
253
+ Measured, not asserted. See [`docs/performance.md`](docs/performance.md).
254
+
255
+ | operation | edaprep | baseline |
256
+ |---|---|---|
257
+ | `Scaler` (standard) | 6.9 ms | sklearn `StandardScaler` 26.7 ms |
258
+ | `MissingValueHandler` (median) | 5.2 ms | sklearn `SimpleImputer` 17.0 ms |
259
+ | `OutlierHandler` (IQR clip) | 25.2 ms | the usual IQR block 41.4 ms |
260
+ | `numeric_block_stats` (20k × 300) | 578 ms | equivalent pandas loop 1056 ms |
261
+ | `AutoPipeline.transform` | 240 ms / 35.6 MiB | `ColumnTransformer` 104 ms / 67.2 MiB |
262
+
263
+ 100,000 rows unless stated. The most instructive result is one that went the other way:
264
+ a hand-written NumPy kernel in this library turned out to be **2.1× slower** than the
265
+ pandas code it replaced, so it was deleted. That story is in `docs/performance.md` §1.
266
+
267
+ No native code. Nothing here is un-vectorisable, and the one place a hand-written kernel
268
+ looked promising was slower than pandas.
269
+
270
+ ---
271
+
272
+ ## Documentation
273
+
274
+ | | |
275
+ |---|---|
276
+ | [Workflow mining](docs/design-rationale.md) | what 13 repositories revealed, and the 9 defects found |
277
+ | [Architecture](docs/architecture.md) | package design, the planner, execution model |
278
+ | [User guide](docs/guide.md) | installation to production, with the train/test workflow |
279
+ | [Performance](docs/performance.md) | benchmarks, method, and what optimisation actually changed |
280
+ | [Extending](docs/extending.md) | custom transformers, rules and backends |
281
+ | [Example](examples/end_to_end.py) | raw dataset to ML-ready, end to end |
282
+
283
+ ---
284
+
285
+ ## Scope
286
+
287
+ **In:** dataset inspection, EDA, data quality, cleaning, missing values, duplicates,
288
+ outliers, dtype inference, categorical encoding, numeric transformation, scaling,
289
+ feature selection, datetime expansion, leakage-safe train/test preparation, pipelines,
290
+ reporting.
291
+
292
+ **Out, deliberately:** model training, resampling, hyperparameter search, NLP,
293
+ forecasting, deep learning, distributed execution. Extension points exist for each
294
+ (`docs/extending.md`), and none is implemented in v1.
295
+
296
+ ## Development
297
+
298
+ ```bash
299
+ pip install -e ".[dev]"
300
+ pytest # 353 tests
301
+ python benchmarks/bench.py
302
+ ```
303
+
304
+ ## Licence
305
+
306
+ MIT.
@@ -0,0 +1,260 @@
1
+ # edaprep
2
+
3
+ [![CI](https://github.com/bijay-odyssey/edaprep/actions/workflows/ci.yml/badge.svg)](https://github.com/bijay-odyssey/edaprep/actions/workflows/ci.yml)
4
+ [![Python](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)](https://pypi.org/project/edaprep/)
5
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
6
+
7
+ Transparent, leakage-safe EDA and ML preprocessing, with an explainable planner.
8
+
9
+ `edaprep` looks at a dataset, works out which preprocessing operations actually apply
10
+ to it, tells you what it intends to do and why, and then does it — fitting every
11
+ statistic on the training data alone.
12
+
13
+ ```python
14
+ import edaprep
15
+
16
+ pipe = edaprep.AutoPipeline(target="churn", model_family="tree", random_state=42)
17
+ pipe.fit(train_df)
18
+ pipe.explain()
19
+
20
+ X_train = pipe.transform(train_df)
21
+ X_test = pipe.transform(test_df)
22
+ ```
23
+
24
+ ```
25
+ income:
26
+ + outliers_report - skew 3.22 is moderate (>= 1.0); IQR fence widened to k=3.0 for the asymmetry
27
+ + impute_median - 2.0% missing; median rather than mean because it is unaffected by
28
+ the skew (3.22) and by outliers
29
+ + transform_log1p - skew 3.22 is moderate and the column is non-negative (min 2576.27),
30
+ so log1p applies and is invertible
31
+ + scale_robust - skew 3.22; robust scaling (median and IQR) rather than standard,
32
+ whose standard deviation is dominated by the tail
33
+
34
+ city:
35
+ + group_rare_categories - 163 levels; those appearing in fewer than 5 rows (1.0%) are
36
+ grouped, since they cannot support a reliable estimate
37
+ + encode_target - 163 levels exceeds the 50-level one-hot ceiling; target encoding is
38
+ used with 5-fold cross-fitting so no row is encoded using its own target
39
+
40
+ customer_id:
41
+ x dropped - identifier: 100.0% of values are distinct, so it cannot generalise beyond
42
+ the rows it was fitted on
43
+ ```
44
+
45
+ ---
46
+
47
+ ## Why it exists
48
+
49
+ It was built by mining common notebook workflows for the EDA and
50
+ preprocessing workflow they have in common — a broad survey of notebook workflows.
51
+ The findings are written up in [`docs/design-rationale.md`](docs/design-rationale.md), and
52
+ they shaped every design decision:
53
+
54
+ - The dtype-based column split (`select_dtypes(include=['int64','float64'])`) appears
55
+ **39 times** and is the largest single source of error: it sends a zip code and a
56
+ temperature down the same path. `edaprep` infers a *semantic* type and reports its
57
+ confidence.
58
+ - The IQR outlier fence is rewritten **12 times**, the z-score fence 6 times, with the
59
+ multiplier drifting between 1.5 and 3.0 for no recorded reason. Both are now single
60
+ parameterised, named, reported operations.
61
+ - **Leakage is easy to introduce.** One fits a `StandardScaler` on the full
62
+ frame, writes the result to CSV, and splits afterwards. `edaprep` makes that
63
+ structurally impossible rather than merely discouraged.
64
+ - Two notebooks independently maintain *parallel preprocessing branches* for tree and
65
+ linear models. That insight became `model_family`, a first-class planning input.
66
+
67
+ ## Installation
68
+
69
+ Not yet published to PyPI. Install from the tagged release:
70
+
71
+ ```bash
72
+ pip install "git+https://github.com/bijay-odyssey/edaprep@v0.1.0"
73
+ pip install "edaprep[visualization] @ git+https://github.com/bijay-odyssey/edaprep@v0.1.0"
74
+ ```
75
+
76
+ Once it is on PyPI, this becomes:
77
+
78
+ ```bash
79
+ pip install edaprep # core: numpy, pandas, scipy
80
+ pip install "edaprep[visualization]" # + matplotlib
81
+ pip install "edaprep[advanced]" # + scikit-learn
82
+ pip install "edaprep[all]"
83
+ ```
84
+
85
+ Python 3.9–3.13, tested on Linux, macOS and Windows.
86
+
87
+ ---
88
+
89
+ ## What it does
90
+
91
+ ### Understand a dataset
92
+
93
+ ```python
94
+ profile = edaprep.profile(df, target="churn")
95
+ print(profile.summary())
96
+ ```
97
+
98
+ ```
99
+ Dataset
100
+ 600 rows x 18 columns
101
+ 300.3 KB in memory
102
+ 628 missing cells (5.81%)
103
+ target: churn (classification, 2 classes, minority/majority ratio 0.232)
104
+
105
+ Semantic types
106
+ numeric 5
107
+ binary 4
108
+ categorical 3
109
+ ...
110
+
111
+ Data-quality findings
112
+ [x] 1 column(s) are almost perfectly associated with the target (>= 0.98). This
113
+ usually means the column encodes the answer: 'leaky'.
114
+ [!] 2 column(s) contain placeholder strings that most likely mean 'missing' but are
115
+ not recognised as NaN: 'workclass', 'occupation'.
116
+ [!] 1 group(s) of identical columns: income=income_copy
117
+ [i] 1 column pair(s) go missing together, which usually means a shared cause:
118
+ income~income_copy (1.00)
119
+ ```
120
+
121
+ ### Explore it
122
+
123
+ ```python
124
+ report = edaprep.EDA(df, target="churn").analyze("standard")
125
+ print(report.summary())
126
+ report.numerical # a DataFrame
127
+ report.to_html("eda.html")
128
+ ```
129
+
130
+ Three levels that differ in work done, not just in what is shown: `quick` skips every
131
+ O(n log n) and O(p²) computation, `standard` adds moments, outliers, correlation and
132
+ target relationships, `deep` adds VIF and significance tests with a
133
+ Benjamini-Hochberg adjustment.
134
+
135
+ ### Prepare it
136
+
137
+ ```python
138
+ pipe = edaprep.AutoPipeline(target="churn", model_family="linear", random_state=42)
139
+ X_train = pipe.fit_transform(train_df)
140
+ X_test = pipe.transform(test_df)
141
+
142
+ pipe.plan_ # the decisions, serialisable and editable
143
+ pipe.report_ # what actually happened, with counts
144
+ pipe.transformations_ # one row per decision, as a DataFrame
145
+ pipe.statistics_ # every learned parameter
146
+ ```
147
+
148
+ Or say exactly what should happen:
149
+
150
+ ```python
151
+ pipe = (
152
+ edaprep.Pipeline(target="churn")
153
+ .flag_missing()
154
+ .handle_outliers(strategy="clip")
155
+ .handle_missing()
156
+ .encode_categorical()
157
+ .scale_numeric()
158
+ )
159
+ ```
160
+
161
+ ### Override anything
162
+
163
+ ```python
164
+ config = edaprep.Config(random_state=42)
165
+ config.column("age").imputation = "mean"
166
+ config.column("income").outlier_strategy = "clip"
167
+ config.column("city").encoding = "frequency"
168
+ config.column("zip").semantic_type = "categorical"
169
+ config.thresholds.skew_heavy = 4.0
170
+
171
+ pipe = edaprep.AutoPipeline(target="churn", config=config)
172
+ ```
173
+
174
+ Overrides are tagged in the plan, so `explain()` marks them as yours rather than
175
+ presenting them as the planner's reasoning.
176
+
177
+ ---
178
+
179
+ ## Design guarantees
180
+
181
+ **No leakage, structurally.** Learned state lives only in attributes written inside
182
+ `fit`; `transform` is a pure function of that state. The property is asserted directly:
183
+ a test transforms a frame whole and then row by row and requires identical output, which
184
+ fails immediately if anything recomputes a statistic at transform time.
185
+
186
+ **Nothing silent.** Dropped columns, imputed values, grouped categories, clipped rows
187
+ and unseen categories are all counted and reported. `edaprep` never calls
188
+ `warnings.filterwarnings`.
189
+
190
+ **Everything explainable.** Every automatic decision carries an English rationale naming
191
+ the measurement behind it. The plan is inert, serialisable data — printable, diffable,
192
+ storable next to a model artefact, and re-executable.
193
+
194
+ **Reproducible.** `random_state` seeds every stochastic step. The report records the
195
+ library version, the configuration, the seed, whether profiling sampled, and every
196
+ learned parameter.
197
+
198
+ **Conservative.** Outliers are reported, not deleted, by default. Duplicate rows are
199
+ reported, not removed — repeated observations are legitimate in transactional data.
200
+ Class imbalance is measured and reported; resampling is a modelling decision that
201
+ belongs after the split, so `edaprep` does not do it.
202
+
203
+ ---
204
+
205
+ ## Performance
206
+
207
+ Measured, not asserted. See [`docs/performance.md`](docs/performance.md).
208
+
209
+ | operation | edaprep | baseline |
210
+ |---|---|---|
211
+ | `Scaler` (standard) | 6.9 ms | sklearn `StandardScaler` 26.7 ms |
212
+ | `MissingValueHandler` (median) | 5.2 ms | sklearn `SimpleImputer` 17.0 ms |
213
+ | `OutlierHandler` (IQR clip) | 25.2 ms | the usual IQR block 41.4 ms |
214
+ | `numeric_block_stats` (20k × 300) | 578 ms | equivalent pandas loop 1056 ms |
215
+ | `AutoPipeline.transform` | 240 ms / 35.6 MiB | `ColumnTransformer` 104 ms / 67.2 MiB |
216
+
217
+ 100,000 rows unless stated. The most instructive result is one that went the other way:
218
+ a hand-written NumPy kernel in this library turned out to be **2.1× slower** than the
219
+ pandas code it replaced, so it was deleted. That story is in `docs/performance.md` §1.
220
+
221
+ No native code. Nothing here is un-vectorisable, and the one place a hand-written kernel
222
+ looked promising was slower than pandas.
223
+
224
+ ---
225
+
226
+ ## Documentation
227
+
228
+ | | |
229
+ |---|---|
230
+ | [Workflow mining](docs/design-rationale.md) | what 13 repositories revealed, and the 9 defects found |
231
+ | [Architecture](docs/architecture.md) | package design, the planner, execution model |
232
+ | [User guide](docs/guide.md) | installation to production, with the train/test workflow |
233
+ | [Performance](docs/performance.md) | benchmarks, method, and what optimisation actually changed |
234
+ | [Extending](docs/extending.md) | custom transformers, rules and backends |
235
+ | [Example](examples/end_to_end.py) | raw dataset to ML-ready, end to end |
236
+
237
+ ---
238
+
239
+ ## Scope
240
+
241
+ **In:** dataset inspection, EDA, data quality, cleaning, missing values, duplicates,
242
+ outliers, dtype inference, categorical encoding, numeric transformation, scaling,
243
+ feature selection, datetime expansion, leakage-safe train/test preparation, pipelines,
244
+ reporting.
245
+
246
+ **Out, deliberately:** model training, resampling, hyperparameter search, NLP,
247
+ forecasting, deep learning, distributed execution. Extension points exist for each
248
+ (`docs/extending.md`), and none is implemented in v1.
249
+
250
+ ## Development
251
+
252
+ ```bash
253
+ pip install -e ".[dev]"
254
+ pytest # 353 tests
255
+ python benchmarks/bench.py
256
+ ```
257
+
258
+ ## Licence
259
+
260
+ MIT.
@@ -0,0 +1,97 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "edaprep"
7
+ dynamic = ["version"]
8
+ description = "Transparent, leakage-safe EDA and ML preprocessing with an explainable planner."
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = { text = "MIT" }
12
+ authors = [{ name = "bijay", email = "bijaybeezoe@gmail.com" }]
13
+ keywords = ["eda", "preprocessing", "data-cleaning", "machine-learning", "pandas"]
14
+ classifiers = [
15
+ "Development Status :: 4 - Beta",
16
+ "Intended Audience :: Science/Research",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Programming Language :: Python :: 3",
19
+ "Programming Language :: Python :: 3.9",
20
+ "Programming Language :: Python :: 3.10",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Topic :: Scientific/Engineering",
25
+ "Typing :: Typed",
26
+ ]
27
+ dependencies = [
28
+ "numpy>=1.21",
29
+ "pandas>=1.5",
30
+ "scipy>=1.7",
31
+ ]
32
+
33
+ [project.optional-dependencies]
34
+ visualization = ["matplotlib>=3.5"]
35
+ advanced = ["scikit-learn>=1.1"]
36
+ arrow = ["pyarrow>=10.0"]
37
+ all = ["matplotlib>=3.5", "scikit-learn>=1.1", "pyarrow>=10.0"]
38
+ dev = [
39
+ "pytest>=7.0",
40
+ "pytest-cov>=4.0",
41
+ "hypothesis>=6.60",
42
+ "scikit-learn>=1.1",
43
+ "matplotlib>=3.5",
44
+ ]
45
+
46
+ [project.urls]
47
+ Documentation = "https://github.com/bijay-odyssey/edaprep#readme"
48
+ Source = "https://github.com/bijay-odyssey/edaprep"
49
+ Changelog = "https://github.com/bijay-odyssey/edaprep/blob/main/CHANGELOG.md"
50
+ Issues = "https://github.com/bijay-odyssey/edaprep/issues"
51
+
52
+ [tool.setuptools.dynamic]
53
+ version = { attr = "edaprep._version.__version__" }
54
+
55
+ [tool.setuptools.packages.find]
56
+ where = ["src"]
57
+
58
+ [tool.setuptools.package-data]
59
+ edaprep = ["py.typed"]
60
+
61
+ [tool.pytest.ini_options]
62
+ testpaths = ["tests"]
63
+ addopts = "-q --strict-markers"
64
+ markers = [
65
+ "slow: long-running tests (deselect with '-m \"not slow\"')",
66
+ "benchmark: performance measurement, not correctness",
67
+ ]
68
+ filterwarnings = ["error::RuntimeWarning"]
69
+
70
+ [tool.ruff]
71
+ line-length = 96
72
+ target-version = "py39"
73
+
74
+ [tool.ruff.lint]
75
+ select = ["E", "F", "W", "I", "UP", "B", "C4", "SIM", "RET"]
76
+ ignore = [
77
+ "B008",
78
+ # PEP 585/604 modernisation (dict[str, int], X | None). Deliberately not adopted:
79
+ # the package supports Python 3.9, where those forms are only legal inside
80
+ # annotations (via `from __future__ import annotations`) and not in the runtime
81
+ # positions this codebase also uses them in -- dataclass field types resolved by
82
+ # `dataclasses.fields`, and `Stage.coerce` argument checks. Mixing the two
83
+ # spellings would be worse than consistently using `typing`.
84
+ "UP006",
85
+ "UP007",
86
+ "UP035",
87
+ "UP045",
88
+ # Quoted annotations are load-bearing here: `Axes` and `Figure` are imported only
89
+ # under TYPE_CHECKING so that matplotlib stays an optional dependency.
90
+ "UP037",
91
+ ]
92
+
93
+ [tool.mypy]
94
+ python_version = "3.9"
95
+ warn_unused_configs = true
96
+ disallow_untyped_defs = false
97
+ ignore_missing_imports = true
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+