smartclean-df 0.1.0__py3-none-any.whl

smartclean_df/__init__.py

@@ -0,0 +1,6 @@
+"""smartclean-df: turn a messy table into a tidy one in one call, and say what changed."""
+from .core import FORWARD_FILL, Cleaner, clean
+from .result import Action, CleanResult
+
+__version__ = "0.1.0"
+__all__ = ["clean", "Cleaner", "CleanResult", "Action", "FORWARD_FILL", "__version__"]

smartclean_df/_io.py

@@ -0,0 +1,90 @@
+"""Accept a DataFrame or a path to a tabular file."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Union
+
+import pandas as pd
+
+__all__ = ["load_frame", "write_frame", "FrameLike"]
+
+FrameLike = Union[pd.DataFrame, str, "os.PathLike[str]"]
+
+
+def _inference_is_lossless(text: pd.Series) -> bool:
+    """True when letting pandas type-infer this column of text changes nothing.
+
+    ``"123"`` becomes ``123`` and renders back as ``"123"``, so inferring it is
+    free. ``"00123"`` becomes ``123`` and renders back as ``"123"`` -- a
+    different identifier -- so that column must stay text and be handed to the
+    cleaning pipeline, which knows to leave zero-padded codes alone.
+    """
+    values = text.dropna()
+    if values.empty:
+        return True
+    try:
+        numbers = pd.to_numeric(values)
+    except (ValueError, TypeError, OverflowError):
+        # pandas will not turn this column into numbers either; nothing to lose
+        return True
+    try:
+        rendered = numbers.astype(str).to_numpy()
+    except (ValueError, TypeError):
+        return False
+    return bool((rendered == values.to_numpy()).all())
+
+
+def _read_text_table(path: Path, *, sep: str) -> pd.DataFrame:
+    """Read a delimited text file without letting pandas destroy any values.
+
+    Reading a file must never be a lossy, unlogged transformation. Columns
+    pandas can type-infer without changing a single value are inferred as
+    usual; every other column is read as text, so that the cleaning pipeline
+    decides what becomes a number, a date or a boolean -- and logs an Action
+    for it -- instead of pandas deciding silently at read time.
+    """
+    text = pd.read_csv(path, sep=sep, dtype=str, keep_default_na=True)
+    as_text = {col: str for col in text.columns if not _inference_is_lossless(text[col])}
+    if not as_text:
+        return pd.read_csv(path, sep=sep)
+    return pd.read_csv(path, sep=sep, dtype=as_text)
+
+
+def load_frame(df_or_path: FrameLike) -> pd.DataFrame:
+    """Return a deep copy of a DataFrame, or read a .csv/.tsv/.parquet file."""
+    if isinstance(df_or_path, pd.DataFrame):
+        return df_or_path.copy(deep=True)
+    if isinstance(df_or_path, (str, os.PathLike)):
+        path = Path(df_or_path)
+        suffix = path.suffix.lower()
+        if suffix in (".csv", ".txt"):
+            return _read_text_table(path, sep=",")
+        if suffix == ".tsv":
+            return _read_text_table(path, sep="\t")
+        if suffix in (".parquet", ".pq"):
+            return pd.read_parquet(path)
+        raise ValueError(
+            f"Unsupported file type {suffix!r} for {path}; expected .csv, .tsv or .parquet"
+        )
+    raise TypeError(
+        "expected a pandas DataFrame or a path to a .csv/.tsv/.parquet file, "
+        f"got {type(df_or_path).__name__}"
+    )
+
+
+def write_frame(df: pd.DataFrame, path: "Union[str, os.PathLike[str]]") -> Path:
+    """Write ``df`` to ``path`` by extension (.csv, .tsv or .parquet)."""
+    target = Path(path)
+    suffix = target.suffix.lower()
+    if suffix in (".csv", ".txt"):
+        df.to_csv(target, index=False)
+    elif suffix == ".tsv":
+        df.to_csv(target, index=False, sep="\t")
+    elif suffix in (".parquet", ".pq"):
+        df.to_parquet(target, index=False)
+    else:
+        raise ValueError(
+            f"Unsupported output type {suffix!r} for {target}; expected .csv, .tsv or .parquet"
+        )
+    return target

smartclean_df/cli.py

@@ -0,0 +1,99 @@
+"""Command line entry point: ``smartclean-df data.csv --output clean.csv``."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import List, Optional
+
+from . import __version__
+from ._io import write_frame
+from .core import MISSING_MODES, OUTLIER_MODES, clean
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="smartclean-df",
+        description=(
+            "Detect and fix missing values, duplicate rows, outliers, dirty column names and "
+            "numbers/dates/booleans stored as text in a table, and report every change."
+        ),
+    )
+    parser.add_argument("path", help="input table (.csv, .tsv or .parquet)")
+    parser.add_argument(
+        "--missing",
+        choices=MISSING_MODES,
+        default="auto",
+        help="auto: fill with median / mode / forward-fill; drop: drop rows with missing values; "
+        "none: leave them (default auto)",
+    )
+    parser.add_argument(
+        "--outliers",
+        choices=OUTLIER_MODES,
+        default="clip",
+        help="clip: winsorize to the IQR bounds; flag: only report; drop: drop the rows; "
+        "none: skip (default clip)",
+    )
+    parser.add_argument(
+        "--iqr-factor",
+        type=float,
+        default=3.0,
+        metavar="K",
+        help="outlier bounds are Q1 - K*IQR and Q3 + K*IQR (default 3.0)",
+    )
+    parser.add_argument("--keep-duplicates", action="store_true", help="do not drop exact duplicate rows")
+    parser.add_argument("--keep-column-names", action="store_true", help="do not normalize column names")
+    parser.add_argument("--no-parse-numbers", action="store_true", help="leave numbers stored as text alone")
+    parser.add_argument("--no-parse-dates", action="store_true", help="leave dates stored as text alone")
+    parser.add_argument("--no-parse-booleans", action="store_true", help="leave booleans stored as text alone")
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="report what would change without changing anything (cannot be combined with --output)",
+    )
+    parser.add_argument("--json", action="store_true", help="print the result as JSON instead of the text summary")
+    parser.add_argument("--output", metavar="FILE", help="write the cleaned table here (.csv, .tsv or .parquet)")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    return parser
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    """Run the command line; returns the process exit status."""
+    for stream in (sys.stdout, sys.stderr):
+        if hasattr(stream, "reconfigure"):
+            stream.reconfigure(encoding="utf-8", errors="replace")
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.dry_run and args.output:
+        parser.error("--dry-run cannot be combined with --output")
+    try:
+        result = clean(
+            args.path,
+            missing=args.missing,
+            outliers=args.outliers,
+            iqr_factor=args.iqr_factor,
+            duplicates=not args.keep_duplicates,
+            normalize_columns=not args.keep_column_names,
+            parse_numbers=not args.no_parse_numbers,
+            parse_dates=not args.no_parse_dates,
+            parse_booleans=not args.no_parse_booleans,
+            dry_run=args.dry_run,
+        )
+        written = write_frame(result.df, args.output) if args.output else None
+    except (ValueError, TypeError, ImportError, OSError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    if args.json:
+        payload = result.to_dict()
+        if written is not None:
+            payload["output"] = str(written)
+        print(json.dumps(payload, indent=2, ensure_ascii=False))
+    else:
+        print(result.summary())
+        if written is not None:
+            print(f"cleaned table written to {written}")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())