qualityclean 0.1.0__tar.gz

qualityclean-0.1.0/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.3
+Name: qualityclean
+Version: 0.1.0
+Summary: Fast, automated data cleaning for Polars DataFrames.
+Keywords: data-cleaning,polars,data-preprocessing,etl,machine-learning,data-science
+Author: Zeo
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: polars>=1.43.0
+Requires-Dist: rich>=15.0.0
+Requires-Python: >=3.11
+Project-URL: Homepage, https://github.com/JustZeo/qualityclean
+Project-URL: Issues, https://github.com/JustZeo/qualityclean/issues
+Project-URL: Repository, https://github.com/JustZeo/qualityclean
+Description-Content-Type: text/markdown
+

qualityclean-0.1.0/pyproject.toml

@@ -0,0 +1,54 @@
+[project]
+name = "qualityclean"
+version = "0.1.0"
+description = "Fast, automated data cleaning for Polars DataFrames."
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+
+authors = [
+    { name = "Zeo" }
+]
+
+keywords = [
+    "data-cleaning",
+    "polars",
+    "data-preprocessing",
+    "etl",
+    "machine-learning",
+    "data-science",
+]
+
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Software Development :: Libraries",
+]
+
+dependencies = [
+    "jinja2>=3.1.6",
+    "polars>=1.43.0",
+    "rich>=15.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/JustZeo/qualityclean"
+Repository = "https://github.com/JustZeo/qualityclean"
+Issues = "https://github.com/JustZeo/qualityclean/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.1.1",
+    "pytest-cov>=7.1.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.8.13,<0.9.0"]
+build-backend = "uv_build"

qualityclean-0.1.0/src/qualityclean/__init__.py

@@ -0,0 +1,15 @@
+from .__version__ import __version__
+from .io.loader import load
+from .io.exporter import export
+from .core.cleaner import clean
+from .report.audit import audit
+from .result import CleanResult
+
+__all__ = [
+    "__version__",
+    "clean",
+    "load",
+    "export",
+    "audit",
+    "CleanResult",
+]

qualityclean-0.1.0/src/qualityclean/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

qualityclean-0.1.0/src/qualityclean/core/cleaner.py

@@ -0,0 +1,23 @@
+import polars as pl
+from qualityclean.core.pipeline import pipeline
+from qualityclean.report.builder import ReportBuilder
+from qualityclean.result import CleanResult
+def clean(
+    df:pl.DataFrame,
+    **kwargs,
+) -> CleanResult:
+    builder = ReportBuilder()
+    builder.start(
+        df,
+        fill_mode=kwargs.get("fill",False)
+    )
+    df = pipeline(
+        df,
+        builder=builder,
+        **kwargs,
+    )
+    builder.finish(df)
+    return CleanResult(
+        df = df,
+        report=builder.build()
+    )

qualityclean-0.1.0/src/qualityclean/core/pipeline.py

@@ -0,0 +1,45 @@
+from time import perf_counter
+
+import polars as pl
+
+from qualityclean.rules.column_rule import ColumnRule
+from qualityclean.rules.datatype_rule import DatatypeRule
+from qualityclean.rules.duplicate_rule import DuplicateRule
+from qualityclean.rules.empty_rule import EmptyRule
+from qualityclean.rules.missing_rule import MissingRule
+from qualityclean.rules.whitespace_rule import WhitespaceRule
+
+
+DEFAULT_PIPELINE = (
+    ColumnRule(),
+    WhitespaceRule(),
+    EmptyRule(),
+    DatatypeRule(),
+    MissingRule(),
+    DuplicateRule(),
+)
+
+
+def pipeline(
+    df: pl.DataFrame,
+    **kwargs,
+) -> pl.DataFrame:
+
+    builder = kwargs.get("builder")
+
+    for rule in DEFAULT_PIPELINE:
+
+        start = perf_counter()
+
+        df = rule.run(
+            df,
+            **kwargs,
+        )
+
+        if builder is not None:
+            builder.record_rule_time(
+                rule.__class__.__name__,
+                perf_counter() - start,
+            )
+
+    return df

qualityclean-0.1.0/src/qualityclean/io/__init__.py

@@ -0,0 +1 @@
+

qualityclean-0.1.0/src/qualityclean/io/exporter.py

@@ -0,0 +1,24 @@
+import polars as pl 
+from pathlib import Path
+from qualityclean.result import CleanResult
+
+
+def export(data:pl.DataFrame,path:str|Path)-> None:
+    """Export a Polars DataFrame to support file format."""
+
+    if isinstance(data,CleanResult):
+        data = data.df
+    
+    path = Path(path)
+    ext = path.suffix.lower()
+
+    if ext == ".csv":
+        data.write_csv(path)
+        return
+    if ext == ".parquet":
+        data.write_parquet(path)
+        return
+
+    raise ValueError(
+        f"Unsupported file extension: {ext}. Expected '.csv' or '.parquet'"
+    )

qualityclean-0.1.0/src/qualityclean/io/loader.py

@@ -0,0 +1,29 @@
+import polars as pl
+from pathlib import Path
+
+
+def load(data: str | Path | pl.DataFrame) -> pl.DataFrame:
+    """Load a CSV, Parquet file, or Polars DataFrame."""
+
+    if isinstance(data, pl.DataFrame):
+        return data
+
+    path = Path(data)
+
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: '{path}'")
+
+    if not path.is_file():
+        raise IsADirectoryError(f"'{path}' is a directory, expected a file.")
+
+    ext = path.suffix.lower()
+
+    if ext == ".csv":
+        return pl.read_csv(path)
+
+    if ext == ".parquet":
+        return pl.read_parquet(path)
+
+    raise ValueError(
+        f"Unsupported file extension: '{ext}'. Expected '.csv' or '.parquet'."
+    )

qualityclean-0.1.0/src/qualityclean/report/audit.py

@@ -0,0 +1,55 @@
+from qualityclean.result import CleanResult
+
+from .printer import print_report
+from .html import export_html
+from .markdown import export_markdown
+from .json import export_json
+
+
+def audit(
+    result: CleanResult,
+    format: str = "print",
+    path: str | None = None,
+) -> None:
+
+
+    if not isinstance(result, CleanResult):
+        raise TypeError(
+            "qc.audit() expects a CleanResult returned by qc.clean().\n\n"
+            "Example:\n"
+            "    result = qc.clean(df)\n"
+            "    qc.audit(result)"
+        )
+
+    report = result.report
+
+    format = format.lower()
+
+    if format == "print":
+        print_report(result.report)
+
+    elif format == "html":
+        if path is None:
+            raise ValueError(
+                "'path' is required when exporting an HTML report."
+            )
+        export_html(result.report, path)
+
+    elif format == "markdown":
+        if path is None:
+            raise ValueError(
+                "'path' is required when exporting a Markdown report."
+            )
+        export_markdown(result.report, path)
+
+    elif format == "json":
+        if path is None:
+            raise ValueError(
+                "'path' is required when exporting a JSON report."
+            )
+        export_json(result.report, path)
+
+    else:
+        raise ValueError(
+            "format must be one of: print, html, markdown, json"
+        )

qualityclean-0.1.0/src/qualityclean/report/builder.py

@@ -0,0 +1,112 @@
+import time
+import platform
+import sys
+import polars as pl
+
+from qualityclean.report.model import Report
+
+
+class ReportBuilder:
+    def __init__(self):
+        self.report = Report()
+        self._start_time = 0.0
+
+    def start(
+        self,
+        df: pl.DataFrame,
+        *,
+        fill_mode: bool = False,
+    ) -> None:
+        """Capture the initial state of the dataset before cleaning."""
+
+        self.report.fill_mode = fill_mode
+        self.report.original_rows = df.height
+        self.report.original_columns = df.width
+        self.report.memory_before = df.estimated_size("mb")
+        self.report.python_version = (
+            f"{sys.version_info.major}."
+            f"{sys.version_info.minor}."
+            f"{sys.version_info.micro}"
+        )
+        self.report.polars_version = pl.__version__
+        self.report.platform = platform.system()
+
+        self.report.original_schema = {
+            column: str(dtype)
+            for column, dtype in df.schema.items()
+        }
+
+        self._start_time = time.perf_counter()
+
+    def finish(
+        self,
+        df: pl.DataFrame,
+    ) -> None:
+        """Capture the final state of the dataset after cleaning."""
+
+        self.report.final_rows = df.height
+        self.report.final_columns = df.width
+        self.report.memory_after = df.estimated_size("mb")
+
+        self.report.final_schema = {
+            column: str(dtype)
+            for column, dtype in df.schema.items()
+        }
+
+        self.report.execution_time = (
+            time.perf_counter() - self._start_time
+        )
+
+    def build(self) -> Report:
+        """Return the completed report."""
+
+        return self.report
+
+    def record_rule_time(
+        self,
+        rule: str,
+        elapsed: float,
+    ) -> None:
+        """Record execution time for a cleaning rule."""
+
+        self.report.rule_timings[rule] = elapsed
+
+    def record_whitespace_fixed(
+        self,
+        count: int,
+    ) -> None:
+        """Record the number of whitespace fixes."""
+
+        self.report.whitespace_fixed += count
+
+    def record_placeholders_converted(
+        self,
+        count: int,
+    ) -> None:
+        """Record the number of placeholders converted."""
+
+        self.report.placeholders_converted += count
+
+    def record_missing_filled(
+        self,
+        count: int,
+    ) -> None:
+        """Record the number of missing values filled."""
+
+        self.report.missing_filled += count
+
+    def record_missing_dropped(
+        self,
+        count: int,
+    ) -> None:
+        """Record the number of missing values dropped."""
+
+        self.report.missing_dropped += count
+
+    def record_duplicates_removed(
+        self,
+        count: int,
+    ) -> None:
+        """Record the number of duplicates removed."""
+
+        self.report.duplicates_removed += count

qualityclean-0.1.0/src/qualityclean/report/html.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from jinja2 import Environment, PackageLoader, select_autoescape
+
+from qualityclean.report.model import Report
+
+
+env = Environment(
+    loader=PackageLoader(
+        "qualityclean.report",
+        "templates",
+    ),
+    autoescape=select_autoescape(["html"]),
+)
+
+
+def render_html(report: Report) -> str:
+    """Render a QualityClean report as HTML."""
+
+    template = env.get_template("report.html")
+
+    return template.render(report=report)
+
+
+def export_html(
+    report: Report,
+    path: str | Path,
+) -> None:
+    """Export a QualityClean report as an HTML file."""
+
+    path = Path(path)
+
+    path.write_text(
+        render_html(report),
+        encoding="utf-8",
+    )

qualityclean-0.1.0/src/qualityclean/report/json.py

@@ -0,0 +1,22 @@
+import json 
+from dataclasses import asdict
+from pathlib import Path
+
+from qualityclean.report.model import Report
+
+def export_json(
+    report: Report,
+    path:str | Path,
+)-> None :
+    """Export a QualityClean report as JSON FILE."""
+    path = Path(path)
+    with path.open(
+        mode="w",
+        encoding="utf-8",
+    ) as file:
+        json.dump(
+            asdict(report),
+            file,
+            indent=4,
+            ensure_ascii=False
+        )

qualityclean-0.1.0/src/qualityclean/report/markdown.py

@@ -0,0 +1,135 @@
+from pathlib import Path
+
+from qualityclean.report.model import Report
+
+
+def render_markdown(
+    report: Report,
+) -> str:
+    """Render a QualityClean report as Markdown."""
+
+    lines: list[str] = []
+
+    lines.append(f"# QualityClean Report v{report.version}")
+    lines.append("")
+
+    # ==========================
+    # Dataset Summary
+    # ==========================
+
+    lines.append("## Dataset Summary")
+    lines.append("")
+    lines.append("| Metric | Value |")
+    lines.append("|-------|------:|")
+    lines.append(f"| Original Rows | {report.original_rows} |")
+    lines.append(f"| Final Rows | {report.final_rows} |")
+    lines.append(f"| Original Columns | {report.original_columns} |")
+    lines.append(f"| Final Columns | {report.final_columns} |")
+    lines.append("")
+
+    # ==========================
+    # Cleaning Summary
+    # ==========================
+
+    lines.append("## Cleaning Summary")
+    lines.append("")
+    lines.append("| Operation | Count |")
+    lines.append("|-----------|------:|")
+    lines.append(
+        f"| Whitespace Fixed | {report.whitespace_fixed} |"
+    )
+    lines.append(
+        f"| Placeholders Converted | {report.placeholders_converted} |"
+    )
+    lines.append(
+        f"| Missing Values Filled | {report.missing_filled} |"
+    )
+    lines.append(
+        f"| Missing Values Dropped | {report.missing_dropped} |"
+    )
+    lines.append(
+        f"| Duplicates Removed | {report.duplicates_removed} |"
+    )
+    lines.append(
+        f"| Datatypes Changed | {report.datatypes_changed} |"
+    )
+    lines.append("")
+
+    # ==========================
+    # Datatype Changes
+    # ==========================
+
+    if report.datatype_changes:
+
+        lines.append("## Datatype Changes")
+        lines.append("")
+        lines.append("| Column | Before | After |")
+        lines.append("|--------|--------|-------|")
+
+        for column, (before, after) in report.datatype_changes.items():
+            lines.append(
+                f"| {column} | {before} | {after} |"
+            )
+
+        lines.append("")
+
+    # ==========================
+    # Original Schema
+    # ==========================
+
+    if report.original_schema:
+
+        lines.append("## Original Schema")
+        lines.append("")
+        lines.append("| Column | Datatype |")
+        lines.append("|--------|----------|")
+
+        for column, dtype in report.original_schema.items():
+            lines.append(f"| {column} | {dtype} |")
+
+        lines.append("")
+
+    # ==========================
+    # Final Schema
+    # ==========================
+
+    if report.final_schema:
+
+        lines.append("## Final Schema")
+        lines.append("")
+        lines.append("| Column | Datatype |")
+        lines.append("|--------|----------|")
+
+        for column, dtype in report.final_schema.items():
+            lines.append(f"| {column} | {dtype} |")
+
+        lines.append("")
+
+    # ==========================
+    # Execution
+    # ==========================
+
+    lines.append("## Execution")
+    lines.append("")
+    lines.append("| Metric | Value |")
+    lines.append("|--------|------:|")
+    lines.append(f"| Fill Mode | {report.fill_mode} |")
+    lines.append(
+        f"| Execution Time | {report.execution_time:.4f} seconds |"
+    )
+
+    return "\n".join(lines)
+
+
+def export_markdown(
+    report: Report,
+    path: str | Path,
+) -> None:
+    """Export a QualityClean report as a Markdown file."""
+
+    path = Path(path)
+
+    path.write_text(
+        render_markdown(report),
+        encoding="utf-8",
+    )

qualityclean-0.1.0/src/qualityclean/report/model.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class Report:
+
+    version: str = "0.1.0"
+
+    fill_mode: bool = False
+
+    original_rows: int = 0
+    final_rows: int = 0
+
+    original_columns: int = 0
+    final_columns: int = 0
+
+    original_schema: dict[str, str] = field(default_factory=dict)
+    final_schema: dict[str, str] = field(default_factory=dict)
+
+    whitespace_fixed: int = 0
+    placeholders_converted: int = 0
+
+    missing_filled: int = 0
+    missing_dropped: int = 0
+
+    duplicates_removed: int = 0
+
+    datatypes_changed: int = 0
+    datatype_changes: dict[str, str] = field(default_factory=dict)
+
+    memory_before: float = 0.0
+    memory_after: float = 0.0
+
+    python_version: str = ""
+    polars_version: str = ""
+    platform: str = ""
+
+
+    rule_timings: dict[str, float] = field(default_factory=dict)
+    execution_time: float = 0.0