smartclean-df 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Pranay Mahendrakar
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,151 @@
1
+ Metadata-Version: 2.5
2
+ Name: smartclean-df
3
+ Version: 0.1.0
4
+ Summary: Automatically detects and fixes missing values, duplicates, outliers, inconsistent formats and dirty columns in tabular data
5
+ Project-URL: Homepage, https://pypi.org/project/smartclean-df/
6
+ Project-URL: Author, https://pypi.org/user/pranaymahendrakar/
7
+ Author: Pranay Mahendrakar
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: data-cleaning,dataframe,duplicates,etl,missing-values,outliers,pandas,preprocessing
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
18
+ Requires-Python: >=3.9
19
+ Requires-Dist: numpy>=1.23
20
+ Requires-Dist: pandas>=1.5
21
+ Provides-Extra: dev
22
+ Requires-Dist: pytest>=7; extra == 'dev'
23
+ Provides-Extra: parquet
24
+ Requires-Dist: pyarrow>=12; extra == 'parquet'
25
+ Description-Content-Type: text/markdown
26
+
27
+ # smartclean-df
28
+
29
+ Turns a messy table into a tidy one in one call: it finds and fixes missing
30
+ values, duplicate rows, outliers, numbers/dates/booleans stored as text, and
31
+ dirty column names, and tells you exactly what it changed.
32
+
33
+ ## Install
34
+
35
+ ```
36
+ pip install smartclean-df
37
+ ```
38
+
39
+ Parquet input needs the optional extra: `pip install "smartclean-df[parquet]"`.
40
+
41
+ ## Quickstart
42
+
43
+ ```python
44
+ import pandas as pd
45
+ from smartclean_df import clean
46
+ df = pd.DataFrame({"Name ": ["Ann", " Bob", "Ann", None], "Age": ["34", "NA", "34", "41"], "Joined": ["2024-01-05", "2024-02-10", "2024-01-05", "-"]})
47
+ result = clean(df)
48
+ print(result.summary())
49
+ print(result.df)
50
+ ```
51
+
52
+ `result.df` is the cleaned copy (the input is never modified), `result.actions`
53
+ lists every change in the order it was made, and `result.summary()` prints it
54
+ as text. `df, actions = clean(df)` also works.
55
+
56
+ ## What it does
57
+
58
+ Every step runs in this order and every change is logged as an `Action`:
59
+
60
+ 1. **Column names** are stripped, inner whitespace collapsed to `_`, and
61
+ lowercased (`"Order ID "` becomes `order_id`); clashes get a `_2` suffix.
62
+ 2. **String cells** are stripped of surrounding whitespace, and missing tokens
63
+ (`""`, `NA`, `N/A`, `null`, `None`, `-`, `?`, `nan`, any case) become `NaN`.
64
+ 3. **Numbers stored as text** are parsed when at least 90% of the non-missing
65
+ values parse: `"1,234"`, `"$12.50"`, `"(300)"`, `"45%"` (the `%` is
66
+ stripped, the value is kept as written: `45.0`), `" 7 "`. Whole-number
67
+ columns become `int64`. Codes with leading zeros (`"00123"`) are left alone.
68
+ Columns that mix numbers and text stay `object`.
69
+ 4. **Dates stored as text** are parsed when at least 90% parse. Day-first
70
+ versus month-first is inferred from the data, and no pandas warnings leak.
71
+ 5. **Booleans stored as text** (`yes/no`, `true/false`, `y/n`, `t/f`, `1/0`
72
+ strings) become `bool`. Numeric columns are never touched by this step.
73
+ 6. **Exact duplicate rows** are dropped, then rows and columns that are
74
+ entirely empty.
75
+ 7. **Missing values** are imputed: median for numeric columns, mode for
76
+ categorical/boolean columns, forward-fill for datetimes. Integer columns
77
+ stay integer when the median is a whole number. `missing="drop"` drops any
78
+ row with a missing value instead; `missing="none"` leaves them.
79
+ 8. **Outliers** in numeric columns are found with the IQR rule
80
+ (`Q1 - k*IQR`, `Q3 + k*IQR`, `k = iqr_factor`, default 3.0). `"clip"`
81
+ winsorizes them to the bounds, `"flag"` only reports them, `"drop"` removes
82
+ the rows, `"none"` skips the step. Columns with zero IQR (constants,
83
+ 0/1 indicators) are never clipped.
84
+
85
+ If the input has a default `RangeIndex` and rows were dropped, the result is
86
+ re-indexed from 0; any other index is kept so rows stay traceable.
87
+
88
+ ## API
89
+
90
+ ```python
91
+ clean(df_or_path, *, missing="auto", outliers="clip", iqr_factor=3.0, duplicates=True,
92
+ normalize_columns=True, parse_numbers=True, parse_dates=True, parse_booleans=True,
93
+ dry_run=False) -> CleanResult
94
+ ```
95
+
96
+ `df_or_path` is a `pandas.DataFrame` or a path to a `.csv`, `.tsv` or
97
+ `.parquet` file. `missing` is `"auto"`, `"drop"` or `"none"`; `outliers` is
98
+ `"clip"`, `"flag"`, `"drop"` or `"none"`. With `dry_run=True` the actions are
99
+ computed and reported but `result.df` is an unchanged copy of the input.
100
+
101
+ `CleanResult`
102
+
103
+ - `.df` - the cleaned `DataFrame` (unchanged copy when `dry_run=True`)
104
+ - `.actions` - `list[Action]`, every change made, in order
105
+ - `.input_shape`, `.output_shape` - `(rows, columns)` before and after
106
+ - `.summary()` - human-readable text
107
+ - `.to_dict()` - JSON-safe dict (`shapes`, `actions`, output `dtypes`)
108
+
109
+ `Action(column, kind, detail, rows_affected)` - `column` is `None` for
110
+ table-wide actions (dropping duplicate rows, dropping rows with missing
111
+ values). `kind` is one of `rename_column`, `strip_whitespace`,
112
+ `missing_tokens`, `parse_numeric`, `parse_datetime`, `parse_boolean`,
113
+ `drop_duplicates`, `drop_empty_rows`, `drop_empty_column`, `impute`,
114
+ `drop_missing_rows`, `clip_outliers`, `flag_outliers`, `drop_outliers`.
115
+
116
+ ```python
117
+ Cleaner(**same options as clean, except dry_run)
118
+ Cleaner.fit(df_or_path) -> Cleaner
119
+ Cleaner.transform(df_or_path, *, dry_run=False) -> CleanResult
120
+ Cleaner.fit_transform(df_or_path, *, dry_run=False) -> CleanResult
121
+ ```
122
+
123
+ `fit` learns which columns to parse (and how), the imputation value of every
124
+ column, the outlier bounds, and which all-empty columns to drop. `transform`
125
+ applies exactly that to new data, so production batches get the same
126
+ treatment as the data you fitted on: learned medians and modes fill new gaps,
127
+ learned bounds clip new outliers, and a column that was parsed as a date is
128
+ parsed as a date again even if the batch is too small to pass the 90% rule.
129
+ Columns not seen during `fit` only get the stateless whitespace / missing
130
+ token cleanup.
131
+
132
+ The library never prints; it logs through `logging.getLogger("smartclean_df")`.
133
+
134
+ ## CLI
135
+
136
+ ```
137
+ smartclean-df data.csv # print the summary of what would change
138
+ smartclean-df data.csv --json # print to_dict() as JSON
139
+ smartclean-df data.csv --output clean.csv # also write the cleaned table (.csv, .tsv, .parquet)
140
+ smartclean-df data.csv --missing drop --outliers flag --iqr-factor 1.5
141
+ smartclean-df data.csv --dry-run # report only, never write changed data
142
+ smartclean-df --help
143
+ ```
144
+
145
+ Flags mirror the Python options: `--missing`, `--outliers`, `--iqr-factor`,
146
+ `--keep-duplicates`, `--keep-column-names`, `--no-parse-numbers`,
147
+ `--no-parse-dates`, `--no-parse-booleans`, `--dry-run`.
148
+
149
+ ## License
150
+
151
+ MIT
@@ -0,0 +1,125 @@
1
+ # smartclean-df
2
+
3
+ Turns a messy table into a tidy one in one call: it finds and fixes missing
4
+ values, duplicate rows, outliers, numbers/dates/booleans stored as text, and
5
+ dirty column names, and tells you exactly what it changed.
6
+
7
+ ## Install
8
+
9
+ ```
10
+ pip install smartclean-df
11
+ ```
12
+
13
+ Parquet input needs the optional extra: `pip install "smartclean-df[parquet]"`.
14
+
15
+ ## Quickstart
16
+
17
+ ```python
18
+ import pandas as pd
19
+ from smartclean_df import clean
20
+ df = pd.DataFrame({"Name ": ["Ann", " Bob", "Ann", None], "Age": ["34", "NA", "34", "41"], "Joined": ["2024-01-05", "2024-02-10", "2024-01-05", "-"]})
21
+ result = clean(df)
22
+ print(result.summary())
23
+ print(result.df)
24
+ ```
25
+
26
+ `result.df` is the cleaned copy (the input is never modified), `result.actions`
27
+ lists every change in the order it was made, and `result.summary()` prints it
28
+ as text. `df, actions = clean(df)` also works.
29
+
30
+ ## What it does
31
+
32
+ Every step runs in this order and every change is logged as an `Action`:
33
+
34
+ 1. **Column names** are stripped, inner whitespace collapsed to `_`, and
35
+ lowercased (`"Order ID "` becomes `order_id`); clashes get a `_2` suffix.
36
+ 2. **String cells** are stripped of surrounding whitespace, and missing tokens
37
+ (`""`, `NA`, `N/A`, `null`, `None`, `-`, `?`, `nan`, any case) become `NaN`.
38
+ 3. **Numbers stored as text** are parsed when at least 90% of the non-missing
39
+ values parse: `"1,234"`, `"$12.50"`, `"(300)"`, `"45%"` (the `%` is
40
+ stripped, the value is kept as written: `45.0`), `" 7 "`. Whole-number
41
+ columns become `int64`. Codes with leading zeros (`"00123"`) are left alone.
42
+ Columns that mix numbers and text stay `object`.
43
+ 4. **Dates stored as text** are parsed when at least 90% parse. Day-first
44
+ versus month-first is inferred from the data, and no pandas warnings leak.
45
+ 5. **Booleans stored as text** (`yes/no`, `true/false`, `y/n`, `t/f`, `1/0`
46
+ strings) become `bool`. Numeric columns are never touched by this step.
47
+ 6. **Exact duplicate rows** are dropped, then rows and columns that are
48
+ entirely empty.
49
+ 7. **Missing values** are imputed: median for numeric columns, mode for
50
+ categorical/boolean columns, forward-fill for datetimes. Integer columns
51
+ stay integer when the median is a whole number. `missing="drop"` drops any
52
+ row with a missing value instead; `missing="none"` leaves them.
53
+ 8. **Outliers** in numeric columns are found with the IQR rule
54
+ (`Q1 - k*IQR`, `Q3 + k*IQR`, `k = iqr_factor`, default 3.0). `"clip"`
55
+ winsorizes them to the bounds, `"flag"` only reports them, `"drop"` removes
56
+ the rows, `"none"` skips the step. Columns with zero IQR (constants,
57
+ 0/1 indicators) are never clipped.
58
+
59
+ If the input has a default `RangeIndex` and rows were dropped, the result is
60
+ re-indexed from 0; any other index is kept so rows stay traceable.
61
+
62
+ ## API
63
+
64
+ ```python
65
+ clean(df_or_path, *, missing="auto", outliers="clip", iqr_factor=3.0, duplicates=True,
66
+ normalize_columns=True, parse_numbers=True, parse_dates=True, parse_booleans=True,
67
+ dry_run=False) -> CleanResult
68
+ ```
69
+
70
+ `df_or_path` is a `pandas.DataFrame` or a path to a `.csv`, `.tsv` or
71
+ `.parquet` file. `missing` is `"auto"`, `"drop"` or `"none"`; `outliers` is
72
+ `"clip"`, `"flag"`, `"drop"` or `"none"`. With `dry_run=True` the actions are
73
+ computed and reported but `result.df` is an unchanged copy of the input.
74
+
75
+ `CleanResult`
76
+
77
+ - `.df` - the cleaned `DataFrame` (unchanged copy when `dry_run=True`)
78
+ - `.actions` - `list[Action]`, every change made, in order
79
+ - `.input_shape`, `.output_shape` - `(rows, columns)` before and after
80
+ - `.summary()` - human-readable text
81
+ - `.to_dict()` - JSON-safe dict (`shapes`, `actions`, output `dtypes`)
82
+
83
+ `Action(column, kind, detail, rows_affected)` - `column` is `None` for
84
+ table-wide actions (dropping duplicate rows, dropping rows with missing
85
+ values). `kind` is one of `rename_column`, `strip_whitespace`,
86
+ `missing_tokens`, `parse_numeric`, `parse_datetime`, `parse_boolean`,
87
+ `drop_duplicates`, `drop_empty_rows`, `drop_empty_column`, `impute`,
88
+ `drop_missing_rows`, `clip_outliers`, `flag_outliers`, `drop_outliers`.
89
+
90
+ ```python
91
+ Cleaner(**same options as clean, except dry_run)
92
+ Cleaner.fit(df_or_path) -> Cleaner
93
+ Cleaner.transform(df_or_path, *, dry_run=False) -> CleanResult
94
+ Cleaner.fit_transform(df_or_path, *, dry_run=False) -> CleanResult
95
+ ```
96
+
97
+ `fit` learns which columns to parse (and how), the imputation value of every
98
+ column, the outlier bounds, and which all-empty columns to drop. `transform`
99
+ applies exactly that to new data, so production batches get the same
100
+ treatment as the data you fitted on: learned medians and modes fill new gaps,
101
+ learned bounds clip new outliers, and a column that was parsed as a date is
102
+ parsed as a date again even if the batch is too small to pass the 90% rule.
103
+ Columns not seen during `fit` only get the stateless whitespace / missing
104
+ token cleanup.
105
+
106
+ The library never prints; it logs through `logging.getLogger("smartclean_df")`.
107
+
108
+ ## CLI
109
+
110
+ ```
111
+ smartclean-df data.csv # print the summary of what would change
112
+ smartclean-df data.csv --json # print to_dict() as JSON
113
+ smartclean-df data.csv --output clean.csv # also write the cleaned table (.csv, .tsv, .parquet)
114
+ smartclean-df data.csv --missing drop --outliers flag --iqr-factor 1.5
115
+ smartclean-df data.csv --dry-run # report only, never write changed data
116
+ smartclean-df --help
117
+ ```
118
+
119
+ Flags mirror the Python options: `--missing`, `--outliers`, `--iqr-factor`,
120
+ `--keep-duplicates`, `--keep-column-names`, `--no-parse-numbers`,
121
+ `--no-parse-dates`, `--no-parse-booleans`, `--dry-run`.
122
+
123
+ ## License
124
+
125
+ MIT
@@ -0,0 +1,59 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.27"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "smartclean-df"
7
+ version = "0.1.0"
8
+ description = "Automatically detects and fixes missing values, duplicates, outliers, inconsistent formats and dirty columns in tabular data"
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Pranay Mahendrakar" }]
14
+ keywords = [
15
+ "pandas",
16
+ "data-cleaning",
17
+ "missing-values",
18
+ "outliers",
19
+ "duplicates",
20
+ "dataframe",
21
+ "preprocessing",
22
+ "etl",
23
+ ]
24
+ classifiers = [
25
+ "Development Status :: 4 - Beta",
26
+ "Intended Audience :: Developers",
27
+ "Intended Audience :: Science/Research",
28
+ "Programming Language :: Python :: 3",
29
+ "Programming Language :: Python :: 3 :: Only",
30
+ "Operating System :: OS Independent",
31
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
32
+ ]
33
+ dependencies = [
34
+ "pandas>=1.5",
35
+ "numpy>=1.23",
36
+ ]
37
+
38
+ [project.optional-dependencies]
39
+ # heavy or niche deps go here, never in `dependencies`
40
+ parquet = ["pyarrow>=12"]
41
+ dev = ["pytest>=7"]
42
+
43
+ [project.scripts]
44
+ smartclean-df = "smartclean_df.cli:main"
45
+
46
+ [project.urls]
47
+ Homepage = "https://pypi.org/project/smartclean-df/"
48
+ Author = "https://pypi.org/user/pranaymahendrakar/"
49
+
50
+ [tool.hatch.build.targets.wheel]
51
+ packages = ["src/smartclean_df"]
52
+
53
+ [tool.pytest.ini_options]
54
+ testpaths = ["tests"]
55
+ filterwarnings = [
56
+ "error::UserWarning",
57
+ "error::FutureWarning",
58
+ "error::RuntimeWarning",
59
+ ]
@@ -0,0 +1,6 @@
1
+ """smartclean-df: turn a messy table into a tidy one in one call, and say what changed."""
2
+ from .core import FORWARD_FILL, Cleaner, clean
3
+ from .result import Action, CleanResult
4
+
5
+ __version__ = "0.1.0"
6
+ __all__ = ["clean", "Cleaner", "CleanResult", "Action", "FORWARD_FILL", "__version__"]
@@ -0,0 +1,90 @@
1
+ """Accept a DataFrame or a path to a tabular file."""
2
+ from __future__ import annotations
3
+
4
+ import os
5
+ from pathlib import Path
6
+ from typing import Union
7
+
8
+ import pandas as pd
9
+
10
+ __all__ = ["load_frame", "write_frame", "FrameLike"]
11
+
12
+ FrameLike = Union[pd.DataFrame, str, "os.PathLike[str]"]
13
+
14
+
15
+ def _inference_is_lossless(text: pd.Series) -> bool:
16
+ """True when letting pandas type-infer this column of text changes nothing.
17
+
18
+ ``"123"`` becomes ``123`` and renders back as ``"123"``, so inferring it is
19
+ free. ``"00123"`` becomes ``123`` and renders back as ``"123"`` -- a
20
+ different identifier -- so that column must stay text and be handed to the
21
+ cleaning pipeline, which knows to leave zero-padded codes alone.
22
+ """
23
+ values = text.dropna()
24
+ if values.empty:
25
+ return True
26
+ try:
27
+ numbers = pd.to_numeric(values)
28
+ except (ValueError, TypeError, OverflowError):
29
+ # pandas will not turn this column into numbers either; nothing to lose
30
+ return True
31
+ try:
32
+ rendered = numbers.astype(str).to_numpy()
33
+ except (ValueError, TypeError):
34
+ return False
35
+ return bool((rendered == values.to_numpy()).all())
36
+
37
+
38
+ def _read_text_table(path: Path, *, sep: str) -> pd.DataFrame:
39
+ """Read a delimited text file without letting pandas destroy any values.
40
+
41
+ Reading a file must never be a lossy, unlogged transformation. Columns
42
+ pandas can type-infer without changing a single value are inferred as
43
+ usual; every other column is read as text, so that the cleaning pipeline
44
+ decides what becomes a number, a date or a boolean -- and logs an Action
45
+ for it -- instead of pandas deciding silently at read time.
46
+ """
47
+ text = pd.read_csv(path, sep=sep, dtype=str, keep_default_na=True)
48
+ as_text = {col: str for col in text.columns if not _inference_is_lossless(text[col])}
49
+ if not as_text:
50
+ return pd.read_csv(path, sep=sep)
51
+ return pd.read_csv(path, sep=sep, dtype=as_text)
52
+
53
+
54
+ def load_frame(df_or_path: FrameLike) -> pd.DataFrame:
55
+ """Return a deep copy of a DataFrame, or read a .csv/.tsv/.parquet file."""
56
+ if isinstance(df_or_path, pd.DataFrame):
57
+ return df_or_path.copy(deep=True)
58
+ if isinstance(df_or_path, (str, os.PathLike)):
59
+ path = Path(df_or_path)
60
+ suffix = path.suffix.lower()
61
+ if suffix in (".csv", ".txt"):
62
+ return _read_text_table(path, sep=",")
63
+ if suffix == ".tsv":
64
+ return _read_text_table(path, sep="\t")
65
+ if suffix in (".parquet", ".pq"):
66
+ return pd.read_parquet(path)
67
+ raise ValueError(
68
+ f"Unsupported file type {suffix!r} for {path}; expected .csv, .tsv or .parquet"
69
+ )
70
+ raise TypeError(
71
+ "expected a pandas DataFrame or a path to a .csv/.tsv/.parquet file, "
72
+ f"got {type(df_or_path).__name__}"
73
+ )
74
+
75
+
76
+ def write_frame(df: pd.DataFrame, path: "Union[str, os.PathLike[str]]") -> Path:
77
+ """Write ``df`` to ``path`` by extension (.csv, .tsv or .parquet)."""
78
+ target = Path(path)
79
+ suffix = target.suffix.lower()
80
+ if suffix in (".csv", ".txt"):
81
+ df.to_csv(target, index=False)
82
+ elif suffix == ".tsv":
83
+ df.to_csv(target, index=False, sep="\t")
84
+ elif suffix in (".parquet", ".pq"):
85
+ df.to_parquet(target, index=False)
86
+ else:
87
+ raise ValueError(
88
+ f"Unsupported output type {suffix!r} for {target}; expected .csv, .tsv or .parquet"
89
+ )
90
+ return target
@@ -0,0 +1,99 @@
1
+ """Command line entry point: ``smartclean-df data.csv --output clean.csv``."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import sys
7
+ from typing import List, Optional
8
+
9
+ from . import __version__
10
+ from ._io import write_frame
11
+ from .core import MISSING_MODES, OUTLIER_MODES, clean
12
+
13
+
14
+ def build_parser() -> argparse.ArgumentParser:
15
+ parser = argparse.ArgumentParser(
16
+ prog="smartclean-df",
17
+ description=(
18
+ "Detect and fix missing values, duplicate rows, outliers, dirty column names and "
19
+ "numbers/dates/booleans stored as text in a table, and report every change."
20
+ ),
21
+ )
22
+ parser.add_argument("path", help="input table (.csv, .tsv or .parquet)")
23
+ parser.add_argument(
24
+ "--missing",
25
+ choices=MISSING_MODES,
26
+ default="auto",
27
+ help="auto: fill with median / mode / forward-fill; drop: drop rows with missing values; "
28
+ "none: leave them (default auto)",
29
+ )
30
+ parser.add_argument(
31
+ "--outliers",
32
+ choices=OUTLIER_MODES,
33
+ default="clip",
34
+ help="clip: winsorize to the IQR bounds; flag: only report; drop: drop the rows; "
35
+ "none: skip (default clip)",
36
+ )
37
+ parser.add_argument(
38
+ "--iqr-factor",
39
+ type=float,
40
+ default=3.0,
41
+ metavar="K",
42
+ help="outlier bounds are Q1 - K*IQR and Q3 + K*IQR (default 3.0)",
43
+ )
44
+ parser.add_argument("--keep-duplicates", action="store_true", help="do not drop exact duplicate rows")
45
+ parser.add_argument("--keep-column-names", action="store_true", help="do not normalize column names")
46
+ parser.add_argument("--no-parse-numbers", action="store_true", help="leave numbers stored as text alone")
47
+ parser.add_argument("--no-parse-dates", action="store_true", help="leave dates stored as text alone")
48
+ parser.add_argument("--no-parse-booleans", action="store_true", help="leave booleans stored as text alone")
49
+ parser.add_argument(
50
+ "--dry-run",
51
+ action="store_true",
52
+ help="report what would change without changing anything (cannot be combined with --output)",
53
+ )
54
+ parser.add_argument("--json", action="store_true", help="print the result as JSON instead of the text summary")
55
+ parser.add_argument("--output", metavar="FILE", help="write the cleaned table here (.csv, .tsv or .parquet)")
56
+ parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
57
+ return parser
58
+
59
+
60
+ def main(argv: Optional[List[str]] = None) -> int:
61
+ """Run the command line; returns the process exit status."""
62
+ for stream in (sys.stdout, sys.stderr):
63
+ if hasattr(stream, "reconfigure"):
64
+ stream.reconfigure(encoding="utf-8", errors="replace")
65
+ parser = build_parser()
66
+ args = parser.parse_args(argv)
67
+ if args.dry_run and args.output:
68
+ parser.error("--dry-run cannot be combined with --output")
69
+ try:
70
+ result = clean(
71
+ args.path,
72
+ missing=args.missing,
73
+ outliers=args.outliers,
74
+ iqr_factor=args.iqr_factor,
75
+ duplicates=not args.keep_duplicates,
76
+ normalize_columns=not args.keep_column_names,
77
+ parse_numbers=not args.no_parse_numbers,
78
+ parse_dates=not args.no_parse_dates,
79
+ parse_booleans=not args.no_parse_booleans,
80
+ dry_run=args.dry_run,
81
+ )
82
+ written = write_frame(result.df, args.output) if args.output else None
83
+ except (ValueError, TypeError, ImportError, OSError) as exc:
84
+ print(f"error: {exc}", file=sys.stderr)
85
+ return 2
86
+ if args.json:
87
+ payload = result.to_dict()
88
+ if written is not None:
89
+ payload["output"] = str(written)
90
+ print(json.dumps(payload, indent=2, ensure_ascii=False))
91
+ else:
92
+ print(result.summary())
93
+ if written is not None:
94
+ print(f"cleaned table written to {written}")
95
+ return 0
96
+
97
+
98
+ if __name__ == "__main__": # pragma: no cover
99
+ sys.exit(main())