smartclean-df 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smartclean_df-0.1.0/LICENSE +21 -0
- smartclean_df-0.1.0/PKG-INFO +151 -0
- smartclean_df-0.1.0/README.md +125 -0
- smartclean_df-0.1.0/pyproject.toml +59 -0
- smartclean_df-0.1.0/src/smartclean_df/__init__.py +6 -0
- smartclean_df-0.1.0/src/smartclean_df/_io.py +90 -0
- smartclean_df-0.1.0/src/smartclean_df/cli.py +99 -0
- smartclean_df-0.1.0/src/smartclean_df/core.py +1083 -0
- smartclean_df-0.1.0/src/smartclean_df/parsing.py +184 -0
- smartclean_df-0.1.0/src/smartclean_df/result.py +102 -0
- smartclean_df-0.1.0/tests/test_clean.py +535 -0
- smartclean_df-0.1.0/tests/test_cleaner.py +128 -0
- smartclean_df-0.1.0/tests/test_cli.py +100 -0
- smartclean_df-0.1.0/tests/test_parsing.py +98 -0
- smartclean_df-0.1.0/tests/test_quickstart.py +63 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Pranay Mahendrakar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: smartclean-df
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Automatically detects and fixes missing values, duplicates, outliers, inconsistent formats and dirty columns in tabular data
|
|
5
|
+
Project-URL: Homepage, https://pypi.org/project/smartclean-df/
|
|
6
|
+
Project-URL: Author, https://pypi.org/user/pranaymahendrakar/
|
|
7
|
+
Author: Pranay Mahendrakar
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: data-cleaning,dataframe,duplicates,etl,missing-values,outliers,pandas,preprocessing
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
+
Requires-Python: >=3.9
|
|
19
|
+
Requires-Dist: numpy>=1.23
|
|
20
|
+
Requires-Dist: pandas>=1.5
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
23
|
+
Provides-Extra: parquet
|
|
24
|
+
Requires-Dist: pyarrow>=12; extra == 'parquet'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# smartclean-df
|
|
28
|
+
|
|
29
|
+
Turns a messy table into a tidy one in one call: it finds and fixes missing
|
|
30
|
+
values, duplicate rows, outliers, numbers/dates/booleans stored as text, and
|
|
31
|
+
dirty column names, and tells you exactly what it changed.
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
pip install smartclean-df
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Parquet input needs the optional extra: `pip install "smartclean-df[parquet]"`.
|
|
40
|
+
|
|
41
|
+
## Quickstart
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
import pandas as pd
|
|
45
|
+
from smartclean_df import clean
|
|
46
|
+
df = pd.DataFrame({"Name ": ["Ann", " Bob", "Ann", None], "Age": ["34", "NA", "34", "41"], "Joined": ["2024-01-05", "2024-02-10", "2024-01-05", "-"]})
|
|
47
|
+
result = clean(df)
|
|
48
|
+
print(result.summary())
|
|
49
|
+
print(result.df)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`result.df` is the cleaned copy (the input is never modified), `result.actions`
|
|
53
|
+
lists every change in the order it was made, and `result.summary()` prints it
|
|
54
|
+
as text. `df, actions = clean(df)` also works.
|
|
55
|
+
|
|
56
|
+
## What it does
|
|
57
|
+
|
|
58
|
+
Every step runs in this order and every change is logged as an `Action`:
|
|
59
|
+
|
|
60
|
+
1. **Column names** are stripped, inner whitespace collapsed to `_`, and
|
|
61
|
+
lowercased (`"Order ID "` becomes `order_id`); clashes get a `_2` suffix.
|
|
62
|
+
2. **String cells** are stripped of surrounding whitespace, and missing tokens
|
|
63
|
+
(`""`, `NA`, `N/A`, `null`, `None`, `-`, `?`, `nan`, any case) become `NaN`.
|
|
64
|
+
3. **Numbers stored as text** are parsed when at least 90% of the non-missing
|
|
65
|
+
values parse: `"1,234"`, `"$12.50"`, `"(300)"`, `"45%"` (the `%` is
|
|
66
|
+
stripped, the value is kept as written: `45.0`), `" 7 "`. Whole-number
|
|
67
|
+
columns become `int64`. Codes with leading zeros (`"00123"`) are left alone.
|
|
68
|
+
Columns that mix numbers and text stay `object`.
|
|
69
|
+
4. **Dates stored as text** are parsed when at least 90% parse. Day-first
|
|
70
|
+
versus month-first is inferred from the data, and no pandas warnings leak.
|
|
71
|
+
5. **Booleans stored as text** (`yes/no`, `true/false`, `y/n`, `t/f`, `1/0`
|
|
72
|
+
strings) become `bool`. Numeric columns are never touched by this step.
|
|
73
|
+
6. **Exact duplicate rows** are dropped, then rows and columns that are
|
|
74
|
+
entirely empty.
|
|
75
|
+
7. **Missing values** are imputed: median for numeric columns, mode for
|
|
76
|
+
categorical/boolean columns, forward-fill for datetimes. Integer columns
|
|
77
|
+
stay integer when the median is a whole number. `missing="drop"` drops any
|
|
78
|
+
row with a missing value instead; `missing="none"` leaves them.
|
|
79
|
+
8. **Outliers** in numeric columns are found with the IQR rule
|
|
80
|
+
(`Q1 - k*IQR`, `Q3 + k*IQR`, `k = iqr_factor`, default 3.0). `"clip"`
|
|
81
|
+
winsorizes them to the bounds, `"flag"` only reports them, `"drop"` removes
|
|
82
|
+
the rows, `"none"` skips the step. Columns with zero IQR (constants,
|
|
83
|
+
0/1 indicators) are never clipped.
|
|
84
|
+
|
|
85
|
+
If the input has a default `RangeIndex` and rows were dropped, the result is
|
|
86
|
+
re-indexed from 0; any other index is kept so rows stay traceable.
|
|
87
|
+
|
|
88
|
+
## API
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
clean(df_or_path, *, missing="auto", outliers="clip", iqr_factor=3.0, duplicates=True,
|
|
92
|
+
normalize_columns=True, parse_numbers=True, parse_dates=True, parse_booleans=True,
|
|
93
|
+
dry_run=False) -> CleanResult
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
`df_or_path` is a `pandas.DataFrame` or a path to a `.csv`, `.tsv` or
|
|
97
|
+
`.parquet` file. `missing` is `"auto"`, `"drop"` or `"none"`; `outliers` is
|
|
98
|
+
`"clip"`, `"flag"`, `"drop"` or `"none"`. With `dry_run=True` the actions are
|
|
99
|
+
computed and reported but `result.df` is an unchanged copy of the input.
|
|
100
|
+
|
|
101
|
+
`CleanResult`
|
|
102
|
+
|
|
103
|
+
- `.df` - the cleaned `DataFrame` (unchanged copy when `dry_run=True`)
|
|
104
|
+
- `.actions` - `list[Action]`, every change made, in order
|
|
105
|
+
- `.input_shape`, `.output_shape` - `(rows, columns)` before and after
|
|
106
|
+
- `.summary()` - human-readable text
|
|
107
|
+
- `.to_dict()` - JSON-safe dict (`shapes`, `actions`, output `dtypes`)
|
|
108
|
+
|
|
109
|
+
`Action(column, kind, detail, rows_affected)` - `column` is `None` for
|
|
110
|
+
table-wide actions (dropping duplicate rows, dropping rows with missing
|
|
111
|
+
values). `kind` is one of `rename_column`, `strip_whitespace`,
|
|
112
|
+
`missing_tokens`, `parse_numeric`, `parse_datetime`, `parse_boolean`,
|
|
113
|
+
`drop_duplicates`, `drop_empty_rows`, `drop_empty_column`, `impute`,
|
|
114
|
+
`drop_missing_rows`, `clip_outliers`, `flag_outliers`, `drop_outliers`.
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
Cleaner(**same options as clean, except dry_run)
|
|
118
|
+
Cleaner.fit(df_or_path) -> Cleaner
|
|
119
|
+
Cleaner.transform(df_or_path, *, dry_run=False) -> CleanResult
|
|
120
|
+
Cleaner.fit_transform(df_or_path, *, dry_run=False) -> CleanResult
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
`fit` learns which columns to parse (and how), the imputation value of every
|
|
124
|
+
column, the outlier bounds, and which all-empty columns to drop. `transform`
|
|
125
|
+
applies exactly that to new data, so production batches get the same
|
|
126
|
+
treatment as the data you fitted on: learned medians and modes fill new gaps,
|
|
127
|
+
learned bounds clip new outliers, and a column that was parsed as a date is
|
|
128
|
+
parsed as a date again even if the batch is too small to pass the 90% rule.
|
|
129
|
+
Columns not seen during `fit` only get the stateless whitespace / missing
|
|
130
|
+
token cleanup.
|
|
131
|
+
|
|
132
|
+
The library never prints; it logs through `logging.getLogger("smartclean_df")`.
|
|
133
|
+
|
|
134
|
+
## CLI
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
smartclean-df data.csv # print the summary of what would change
|
|
138
|
+
smartclean-df data.csv --json # print to_dict() as JSON
|
|
139
|
+
smartclean-df data.csv --output clean.csv # also write the cleaned table (.csv, .tsv, .parquet)
|
|
140
|
+
smartclean-df data.csv --missing drop --outliers flag --iqr-factor 1.5
|
|
141
|
+
smartclean-df data.csv --dry-run # report only, never write changed data
|
|
142
|
+
smartclean-df --help
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Flags mirror the Python options: `--missing`, `--outliers`, `--iqr-factor`,
|
|
146
|
+
`--keep-duplicates`, `--keep-column-names`, `--no-parse-numbers`,
|
|
147
|
+
`--no-parse-dates`, `--no-parse-booleans`, `--dry-run`.
|
|
148
|
+
|
|
149
|
+
## License
|
|
150
|
+
|
|
151
|
+
MIT
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# smartclean-df
|
|
2
|
+
|
|
3
|
+
Turns a messy table into a tidy one in one call: it finds and fixes missing
|
|
4
|
+
values, duplicate rows, outliers, numbers/dates/booleans stored as text, and
|
|
5
|
+
dirty column names, and tells you exactly what it changed.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
pip install smartclean-df
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Parquet input needs the optional extra: `pip install "smartclean-df[parquet]"`.
|
|
14
|
+
|
|
15
|
+
## Quickstart
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
import pandas as pd
|
|
19
|
+
from smartclean_df import clean
|
|
20
|
+
df = pd.DataFrame({"Name ": ["Ann", " Bob", "Ann", None], "Age": ["34", "NA", "34", "41"], "Joined": ["2024-01-05", "2024-02-10", "2024-01-05", "-"]})
|
|
21
|
+
result = clean(df)
|
|
22
|
+
print(result.summary())
|
|
23
|
+
print(result.df)
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
`result.df` is the cleaned copy (the input is never modified), `result.actions`
|
|
27
|
+
lists every change in the order it was made, and `result.summary()` prints it
|
|
28
|
+
as text. `df, actions = clean(df)` also works.
|
|
29
|
+
|
|
30
|
+
## What it does
|
|
31
|
+
|
|
32
|
+
Every step runs in this order and every change is logged as an `Action`:
|
|
33
|
+
|
|
34
|
+
1. **Column names** are stripped, inner whitespace collapsed to `_`, and
|
|
35
|
+
lowercased (`"Order ID "` becomes `order_id`); clashes get a `_2` suffix.
|
|
36
|
+
2. **String cells** are stripped of surrounding whitespace, and missing tokens
|
|
37
|
+
(`""`, `NA`, `N/A`, `null`, `None`, `-`, `?`, `nan`, any case) become `NaN`.
|
|
38
|
+
3. **Numbers stored as text** are parsed when at least 90% of the non-missing
|
|
39
|
+
values parse: `"1,234"`, `"$12.50"`, `"(300)"`, `"45%"` (the `%` is
|
|
40
|
+
stripped, the value is kept as written: `45.0`), `" 7 "`. Whole-number
|
|
41
|
+
columns become `int64`. Codes with leading zeros (`"00123"`) are left alone.
|
|
42
|
+
Columns that mix numbers and text stay `object`.
|
|
43
|
+
4. **Dates stored as text** are parsed when at least 90% parse. Day-first
|
|
44
|
+
versus month-first is inferred from the data, and no pandas warnings leak.
|
|
45
|
+
5. **Booleans stored as text** (`yes/no`, `true/false`, `y/n`, `t/f`, `1/0`
|
|
46
|
+
strings) become `bool`. Numeric columns are never touched by this step.
|
|
47
|
+
6. **Exact duplicate rows** are dropped, then rows and columns that are
|
|
48
|
+
entirely empty.
|
|
49
|
+
7. **Missing values** are imputed: median for numeric columns, mode for
|
|
50
|
+
categorical/boolean columns, forward-fill for datetimes. Integer columns
|
|
51
|
+
stay integer when the median is a whole number. `missing="drop"` drops any
|
|
52
|
+
row with a missing value instead; `missing="none"` leaves them.
|
|
53
|
+
8. **Outliers** in numeric columns are found with the IQR rule
|
|
54
|
+
(`Q1 - k*IQR`, `Q3 + k*IQR`, `k = iqr_factor`, default 3.0). `"clip"`
|
|
55
|
+
winsorizes them to the bounds, `"flag"` only reports them, `"drop"` removes
|
|
56
|
+
the rows, `"none"` skips the step. Columns with zero IQR (constants,
|
|
57
|
+
0/1 indicators) are never clipped.
|
|
58
|
+
|
|
59
|
+
If the input has a default `RangeIndex` and rows were dropped, the result is
|
|
60
|
+
re-indexed from 0; any other index is kept so rows stay traceable.
|
|
61
|
+
|
|
62
|
+
## API
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
clean(df_or_path, *, missing="auto", outliers="clip", iqr_factor=3.0, duplicates=True,
|
|
66
|
+
normalize_columns=True, parse_numbers=True, parse_dates=True, parse_booleans=True,
|
|
67
|
+
dry_run=False) -> CleanResult
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`df_or_path` is a `pandas.DataFrame` or a path to a `.csv`, `.tsv` or
|
|
71
|
+
`.parquet` file. `missing` is `"auto"`, `"drop"` or `"none"`; `outliers` is
|
|
72
|
+
`"clip"`, `"flag"`, `"drop"` or `"none"`. With `dry_run=True` the actions are
|
|
73
|
+
computed and reported but `result.df` is an unchanged copy of the input.
|
|
74
|
+
|
|
75
|
+
`CleanResult`
|
|
76
|
+
|
|
77
|
+
- `.df` - the cleaned `DataFrame` (unchanged copy when `dry_run=True`)
|
|
78
|
+
- `.actions` - `list[Action]`, every change made, in order
|
|
79
|
+
- `.input_shape`, `.output_shape` - `(rows, columns)` before and after
|
|
80
|
+
- `.summary()` - human-readable text
|
|
81
|
+
- `.to_dict()` - JSON-safe dict (`shapes`, `actions`, output `dtypes`)
|
|
82
|
+
|
|
83
|
+
`Action(column, kind, detail, rows_affected)` - `column` is `None` for
|
|
84
|
+
table-wide actions (dropping duplicate rows, dropping rows with missing
|
|
85
|
+
values). `kind` is one of `rename_column`, `strip_whitespace`,
|
|
86
|
+
`missing_tokens`, `parse_numeric`, `parse_datetime`, `parse_boolean`,
|
|
87
|
+
`drop_duplicates`, `drop_empty_rows`, `drop_empty_column`, `impute`,
|
|
88
|
+
`drop_missing_rows`, `clip_outliers`, `flag_outliers`, `drop_outliers`.
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
Cleaner(**same options as clean, except dry_run)
|
|
92
|
+
Cleaner.fit(df_or_path) -> Cleaner
|
|
93
|
+
Cleaner.transform(df_or_path, *, dry_run=False) -> CleanResult
|
|
94
|
+
Cleaner.fit_transform(df_or_path, *, dry_run=False) -> CleanResult
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`fit` learns which columns to parse (and how), the imputation value of every
|
|
98
|
+
column, the outlier bounds, and which all-empty columns to drop. `transform`
|
|
99
|
+
applies exactly that to new data, so production batches get the same
|
|
100
|
+
treatment as the data you fitted on: learned medians and modes fill new gaps,
|
|
101
|
+
learned bounds clip new outliers, and a column that was parsed as a date is
|
|
102
|
+
parsed as a date again even if the batch is too small to pass the 90% rule.
|
|
103
|
+
Columns not seen during `fit` only get the stateless whitespace / missing
|
|
104
|
+
token cleanup.
|
|
105
|
+
|
|
106
|
+
The library never prints; it logs through `logging.getLogger("smartclean_df")`.
|
|
107
|
+
|
|
108
|
+
## CLI
|
|
109
|
+
|
|
110
|
+
```
|
|
111
|
+
smartclean-df data.csv # print the summary of what would change
|
|
112
|
+
smartclean-df data.csv --json # print to_dict() as JSON
|
|
113
|
+
smartclean-df data.csv --output clean.csv # also write the cleaned table (.csv, .tsv, .parquet)
|
|
114
|
+
smartclean-df data.csv --missing drop --outliers flag --iqr-factor 1.5
|
|
115
|
+
smartclean-df data.csv --dry-run # report only, never write changed data
|
|
116
|
+
smartclean-df --help
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Flags mirror the Python options: `--missing`, `--outliers`, `--iqr-factor`,
|
|
120
|
+
`--keep-duplicates`, `--keep-column-names`, `--no-parse-numbers`,
|
|
121
|
+
`--no-parse-dates`, `--no-parse-booleans`, `--dry-run`.
|
|
122
|
+
|
|
123
|
+
## License
|
|
124
|
+
|
|
125
|
+
MIT
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "smartclean-df"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Automatically detects and fixes missing values, duplicates, outliers, inconsistent formats and dirty columns in tabular data"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Pranay Mahendrakar" }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"pandas",
|
|
16
|
+
"data-cleaning",
|
|
17
|
+
"missing-values",
|
|
18
|
+
"outliers",
|
|
19
|
+
"duplicates",
|
|
20
|
+
"dataframe",
|
|
21
|
+
"preprocessing",
|
|
22
|
+
"etl",
|
|
23
|
+
]
|
|
24
|
+
classifiers = [
|
|
25
|
+
"Development Status :: 4 - Beta",
|
|
26
|
+
"Intended Audience :: Developers",
|
|
27
|
+
"Intended Audience :: Science/Research",
|
|
28
|
+
"Programming Language :: Python :: 3",
|
|
29
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
30
|
+
"Operating System :: OS Independent",
|
|
31
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
32
|
+
]
|
|
33
|
+
dependencies = [
|
|
34
|
+
"pandas>=1.5",
|
|
35
|
+
"numpy>=1.23",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
[project.optional-dependencies]
|
|
39
|
+
# heavy or niche deps go here, never in `dependencies`
|
|
40
|
+
parquet = ["pyarrow>=12"]
|
|
41
|
+
dev = ["pytest>=7"]
|
|
42
|
+
|
|
43
|
+
[project.scripts]
|
|
44
|
+
smartclean-df = "smartclean_df.cli:main"
|
|
45
|
+
|
|
46
|
+
[project.urls]
|
|
47
|
+
Homepage = "https://pypi.org/project/smartclean-df/"
|
|
48
|
+
Author = "https://pypi.org/user/pranaymahendrakar/"
|
|
49
|
+
|
|
50
|
+
[tool.hatch.build.targets.wheel]
|
|
51
|
+
packages = ["src/smartclean_df"]
|
|
52
|
+
|
|
53
|
+
[tool.pytest.ini_options]
|
|
54
|
+
testpaths = ["tests"]
|
|
55
|
+
filterwarnings = [
|
|
56
|
+
"error::UserWarning",
|
|
57
|
+
"error::FutureWarning",
|
|
58
|
+
"error::RuntimeWarning",
|
|
59
|
+
]
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""smartclean-df: turn a messy table into a tidy one in one call, and say what changed."""
|
|
2
|
+
from .core import FORWARD_FILL, Cleaner, clean
|
|
3
|
+
from .result import Action, CleanResult
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
__all__ = ["clean", "Cleaner", "CleanResult", "Action", "FORWARD_FILL", "__version__"]
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Accept a DataFrame or a path to a tabular file."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Union
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
__all__ = ["load_frame", "write_frame", "FrameLike"]
|
|
11
|
+
|
|
12
|
+
FrameLike = Union[pd.DataFrame, str, "os.PathLike[str]"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _inference_is_lossless(text: pd.Series) -> bool:
|
|
16
|
+
"""True when letting pandas type-infer this column of text changes nothing.
|
|
17
|
+
|
|
18
|
+
``"123"`` becomes ``123`` and renders back as ``"123"``, so inferring it is
|
|
19
|
+
free. ``"00123"`` becomes ``123`` and renders back as ``"123"`` -- a
|
|
20
|
+
different identifier -- so that column must stay text and be handed to the
|
|
21
|
+
cleaning pipeline, which knows to leave zero-padded codes alone.
|
|
22
|
+
"""
|
|
23
|
+
values = text.dropna()
|
|
24
|
+
if values.empty:
|
|
25
|
+
return True
|
|
26
|
+
try:
|
|
27
|
+
numbers = pd.to_numeric(values)
|
|
28
|
+
except (ValueError, TypeError, OverflowError):
|
|
29
|
+
# pandas will not turn this column into numbers either; nothing to lose
|
|
30
|
+
return True
|
|
31
|
+
try:
|
|
32
|
+
rendered = numbers.astype(str).to_numpy()
|
|
33
|
+
except (ValueError, TypeError):
|
|
34
|
+
return False
|
|
35
|
+
return bool((rendered == values.to_numpy()).all())
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _read_text_table(path: Path, *, sep: str) -> pd.DataFrame:
|
|
39
|
+
"""Read a delimited text file without letting pandas destroy any values.
|
|
40
|
+
|
|
41
|
+
Reading a file must never be a lossy, unlogged transformation. Columns
|
|
42
|
+
pandas can type-infer without changing a single value are inferred as
|
|
43
|
+
usual; every other column is read as text, so that the cleaning pipeline
|
|
44
|
+
decides what becomes a number, a date or a boolean -- and logs an Action
|
|
45
|
+
for it -- instead of pandas deciding silently at read time.
|
|
46
|
+
"""
|
|
47
|
+
text = pd.read_csv(path, sep=sep, dtype=str, keep_default_na=True)
|
|
48
|
+
as_text = {col: str for col in text.columns if not _inference_is_lossless(text[col])}
|
|
49
|
+
if not as_text:
|
|
50
|
+
return pd.read_csv(path, sep=sep)
|
|
51
|
+
return pd.read_csv(path, sep=sep, dtype=as_text)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def load_frame(df_or_path: FrameLike) -> pd.DataFrame:
|
|
55
|
+
"""Return a deep copy of a DataFrame, or read a .csv/.tsv/.parquet file."""
|
|
56
|
+
if isinstance(df_or_path, pd.DataFrame):
|
|
57
|
+
return df_or_path.copy(deep=True)
|
|
58
|
+
if isinstance(df_or_path, (str, os.PathLike)):
|
|
59
|
+
path = Path(df_or_path)
|
|
60
|
+
suffix = path.suffix.lower()
|
|
61
|
+
if suffix in (".csv", ".txt"):
|
|
62
|
+
return _read_text_table(path, sep=",")
|
|
63
|
+
if suffix == ".tsv":
|
|
64
|
+
return _read_text_table(path, sep="\t")
|
|
65
|
+
if suffix in (".parquet", ".pq"):
|
|
66
|
+
return pd.read_parquet(path)
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"Unsupported file type {suffix!r} for {path}; expected .csv, .tsv or .parquet"
|
|
69
|
+
)
|
|
70
|
+
raise TypeError(
|
|
71
|
+
"expected a pandas DataFrame or a path to a .csv/.tsv/.parquet file, "
|
|
72
|
+
f"got {type(df_or_path).__name__}"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def write_frame(df: pd.DataFrame, path: "Union[str, os.PathLike[str]]") -> Path:
|
|
77
|
+
"""Write ``df`` to ``path`` by extension (.csv, .tsv or .parquet)."""
|
|
78
|
+
target = Path(path)
|
|
79
|
+
suffix = target.suffix.lower()
|
|
80
|
+
if suffix in (".csv", ".txt"):
|
|
81
|
+
df.to_csv(target, index=False)
|
|
82
|
+
elif suffix == ".tsv":
|
|
83
|
+
df.to_csv(target, index=False, sep="\t")
|
|
84
|
+
elif suffix in (".parquet", ".pq"):
|
|
85
|
+
df.to_parquet(target, index=False)
|
|
86
|
+
else:
|
|
87
|
+
raise ValueError(
|
|
88
|
+
f"Unsupported output type {suffix!r} for {target}; expected .csv, .tsv or .parquet"
|
|
89
|
+
)
|
|
90
|
+
return target
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Command line entry point: ``smartclean-df data.csv --output clean.csv``."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from typing import List, Optional
|
|
8
|
+
|
|
9
|
+
from . import __version__
|
|
10
|
+
from ._io import write_frame
|
|
11
|
+
from .core import MISSING_MODES, OUTLIER_MODES, clean
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
15
|
+
parser = argparse.ArgumentParser(
|
|
16
|
+
prog="smartclean-df",
|
|
17
|
+
description=(
|
|
18
|
+
"Detect and fix missing values, duplicate rows, outliers, dirty column names and "
|
|
19
|
+
"numbers/dates/booleans stored as text in a table, and report every change."
|
|
20
|
+
),
|
|
21
|
+
)
|
|
22
|
+
parser.add_argument("path", help="input table (.csv, .tsv or .parquet)")
|
|
23
|
+
parser.add_argument(
|
|
24
|
+
"--missing",
|
|
25
|
+
choices=MISSING_MODES,
|
|
26
|
+
default="auto",
|
|
27
|
+
help="auto: fill with median / mode / forward-fill; drop: drop rows with missing values; "
|
|
28
|
+
"none: leave them (default auto)",
|
|
29
|
+
)
|
|
30
|
+
parser.add_argument(
|
|
31
|
+
"--outliers",
|
|
32
|
+
choices=OUTLIER_MODES,
|
|
33
|
+
default="clip",
|
|
34
|
+
help="clip: winsorize to the IQR bounds; flag: only report; drop: drop the rows; "
|
|
35
|
+
"none: skip (default clip)",
|
|
36
|
+
)
|
|
37
|
+
parser.add_argument(
|
|
38
|
+
"--iqr-factor",
|
|
39
|
+
type=float,
|
|
40
|
+
default=3.0,
|
|
41
|
+
metavar="K",
|
|
42
|
+
help="outlier bounds are Q1 - K*IQR and Q3 + K*IQR (default 3.0)",
|
|
43
|
+
)
|
|
44
|
+
parser.add_argument("--keep-duplicates", action="store_true", help="do not drop exact duplicate rows")
|
|
45
|
+
parser.add_argument("--keep-column-names", action="store_true", help="do not normalize column names")
|
|
46
|
+
parser.add_argument("--no-parse-numbers", action="store_true", help="leave numbers stored as text alone")
|
|
47
|
+
parser.add_argument("--no-parse-dates", action="store_true", help="leave dates stored as text alone")
|
|
48
|
+
parser.add_argument("--no-parse-booleans", action="store_true", help="leave booleans stored as text alone")
|
|
49
|
+
parser.add_argument(
|
|
50
|
+
"--dry-run",
|
|
51
|
+
action="store_true",
|
|
52
|
+
help="report what would change without changing anything (cannot be combined with --output)",
|
|
53
|
+
)
|
|
54
|
+
parser.add_argument("--json", action="store_true", help="print the result as JSON instead of the text summary")
|
|
55
|
+
parser.add_argument("--output", metavar="FILE", help="write the cleaned table here (.csv, .tsv or .parquet)")
|
|
56
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
57
|
+
return parser
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
61
|
+
"""Run the command line; returns the process exit status."""
|
|
62
|
+
for stream in (sys.stdout, sys.stderr):
|
|
63
|
+
if hasattr(stream, "reconfigure"):
|
|
64
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
65
|
+
parser = build_parser()
|
|
66
|
+
args = parser.parse_args(argv)
|
|
67
|
+
if args.dry_run and args.output:
|
|
68
|
+
parser.error("--dry-run cannot be combined with --output")
|
|
69
|
+
try:
|
|
70
|
+
result = clean(
|
|
71
|
+
args.path,
|
|
72
|
+
missing=args.missing,
|
|
73
|
+
outliers=args.outliers,
|
|
74
|
+
iqr_factor=args.iqr_factor,
|
|
75
|
+
duplicates=not args.keep_duplicates,
|
|
76
|
+
normalize_columns=not args.keep_column_names,
|
|
77
|
+
parse_numbers=not args.no_parse_numbers,
|
|
78
|
+
parse_dates=not args.no_parse_dates,
|
|
79
|
+
parse_booleans=not args.no_parse_booleans,
|
|
80
|
+
dry_run=args.dry_run,
|
|
81
|
+
)
|
|
82
|
+
written = write_frame(result.df, args.output) if args.output else None
|
|
83
|
+
except (ValueError, TypeError, ImportError, OSError) as exc:
|
|
84
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
85
|
+
return 2
|
|
86
|
+
if args.json:
|
|
87
|
+
payload = result.to_dict()
|
|
88
|
+
if written is not None:
|
|
89
|
+
payload["output"] = str(written)
|
|
90
|
+
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
91
|
+
else:
|
|
92
|
+
print(result.summary())
|
|
93
|
+
if written is not None:
|
|
94
|
+
print(f"cleaned table written to {written}")
|
|
95
|
+
return 0
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
if __name__ == "__main__": # pragma: no cover
|
|
99
|
+
sys.exit(main())
|