datawash 0.2.0__py3-none-any.whl

datawash/__init__.py

@@ -0,0 +1,9 @@
+"""Datawash - Intelligent data quality analysis and cleaning."""
+
+from __future__ import annotations
+
+from datawash.core.report import analyze
+
+__version__ = "0.2.0"
+
+__all__ = ["analyze", "__version__"]

datawash/adapters/__init__.py

@@ -0,0 +1,12 @@
+"""Data adapters for loading and saving DataFrames."""
+
+# Import adapters to trigger registration
+from datawash.adapters import (
+    csv_adapter,  # noqa: F401
+    excel_adapter,  # noqa: F401
+    json_adapter,  # noqa: F401
+    parquet_adapter,  # noqa: F401
+)
+from datawash.adapters.base import load_dataframe
+
+__all__ = ["load_dataframe"]

datawash/adapters/base.py

@@ -0,0 +1,66 @@
+"""Base adapter interface and loader dispatch."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any, Optional, Protocol
+
+import pandas as pd
+
+from datawash.core.exceptions import AdapterError
+
+logger = logging.getLogger(__name__)
+
+
+class DataAdapter(Protocol):
+    """Protocol for data adapters."""
+
+    def read(self, path: Path, **kwargs: Any) -> pd.DataFrame: ...
+    def write(self, df: pd.DataFrame, path: Path, **kwargs: Any) -> None: ...
+
+
+_ADAPTERS: dict[str, DataAdapter] = {}
+
+
+def register_adapter(extension: str, adapter: DataAdapter) -> None:
+    _ADAPTERS[extension] = adapter
+
+
+def load_dataframe(
+    source: str | Path, format: Optional[str] = None, **kwargs: Any
+) -> pd.DataFrame:
+    """Load a DataFrame from a file path.
+
+    Args:
+        source: Path to the data file.
+        format: Force file format. Auto-detected from extension if None.
+        **kwargs: Passed to the adapter's read method.
+
+    Returns:
+        Loaded DataFrame.
+
+    Raises:
+        AdapterError: If the file cannot be loaded.
+    """
+    path = Path(source)
+    if not path.exists():
+        raise AdapterError(f"File not found: {path}")
+
+    ext = format or path.suffix.lstrip(".")
+    adapter = _ADAPTERS.get(ext)
+    if adapter is None:
+        raise AdapterError(
+            f"Unsupported format: '{ext}'. "
+            f"Supported formats: {', '.join(sorted(_ADAPTERS.keys()))}"
+        )
+
+    try:
+        logger.info("Loading %s with %s adapter", path, ext)
+        df = adapter.read(path, **kwargs)
+        logger.info("Loaded %d rows, %d columns", len(df), len(df.columns))
+        return df
+    except AdapterError:
+        raise
+    except Exception as e:
+        raise AdapterError(f"Failed to read {path}: {e}") from e

datawash/adapters/csv_adapter.py

@@ -0,0 +1,23 @@
+"""CSV file adapter."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from datawash.adapters.base import register_adapter
+
+
+class CsvAdapter:
+    def read(self, path: Path, **kwargs: Any) -> pd.DataFrame:
+        return pd.read_csv(path, **kwargs)
+
+    def write(self, df: pd.DataFrame, path: Path, **kwargs: Any) -> None:
+        df.to_csv(path, index=False, **kwargs)
+
+
+_adapter = CsvAdapter()
+register_adapter("csv", _adapter)
+register_adapter("tsv", _adapter)  # TSV uses same adapter with sep='\t'

datawash/adapters/excel_adapter.py

@@ -0,0 +1,36 @@
+"""Excel file adapter (requires openpyxl)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from datawash.adapters.base import register_adapter
+from datawash.core.exceptions import AdapterError
+
+
+class ExcelAdapter:
+    def read(self, path: Path, **kwargs: Any) -> pd.DataFrame:
+        try:
+            return pd.read_excel(path, **kwargs)
+        except ImportError:
+            raise AdapterError(
+                "Excel support requires openpyxl. "
+                "Install with: pip install datawash[formats]"
+            )
+
+    def write(self, df: pd.DataFrame, path: Path, **kwargs: Any) -> None:
+        try:
+            df.to_excel(path, index=False, **kwargs)
+        except ImportError:
+            raise AdapterError(
+                "Excel support requires openpyxl. "
+                "Install with: pip install datawash[formats]"
+            )
+
+
+_adapter = ExcelAdapter()
+register_adapter("xlsx", _adapter)
+register_adapter("xls", _adapter)

datawash/adapters/json_adapter.py

@@ -0,0 +1,21 @@
+"""JSON file adapter."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from datawash.adapters.base import register_adapter
+
+
+class JsonAdapter:
+    def read(self, path: Path, **kwargs: Any) -> pd.DataFrame:
+        return pd.read_json(path, **kwargs)
+
+    def write(self, df: pd.DataFrame, path: Path, **kwargs: Any) -> None:
+        df.to_json(path, orient="records", indent=2, **kwargs)
+
+
+register_adapter("json", JsonAdapter())

datawash/adapters/parquet_adapter.py

@@ -0,0 +1,34 @@
+"""Parquet file adapter (requires pyarrow)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from datawash.adapters.base import register_adapter
+from datawash.core.exceptions import AdapterError
+
+
+class ParquetAdapter:
+    def read(self, path: Path, **kwargs: Any) -> pd.DataFrame:
+        try:
+            return pd.read_parquet(path, **kwargs)
+        except ImportError:
+            raise AdapterError(
+                "Parquet support requires pyarrow. "
+                "Install with: pip install datawash[formats]"
+            )
+
+    def write(self, df: pd.DataFrame, path: Path, **kwargs: Any) -> None:
+        try:
+            df.to_parquet(path, index=False, **kwargs)
+        except ImportError:
+            raise AdapterError(
+                "Parquet support requires pyarrow. "
+                "Install with: pip install datawash[formats]"
+            )
+
+
+register_adapter("parquet", ParquetAdapter())

datawash/cli/formatters.py

@@ -0,0 +1,110 @@
+"""Rich output formatting for CLI."""
+
+from __future__ import annotations
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+from datawash.core.models import DatasetProfile, Finding, Suggestion
+
+console = Console()
+
+SEVERITY_COLORS = {"high": "red", "medium": "yellow", "low": "green"}
+
+
+def format_profile(profile: DatasetProfile) -> None:
+    """Print dataset profile."""
+    console.print(
+        Panel(
+            f"[bold]{profile.row_count}[/] rows x "
+            f"[bold]{profile.column_count}[/] columns | "
+            f"Memory: [bold]"
+            f"{profile.memory_bytes / 1024 / 1024:.1f} MB[/] | "
+            f"Dupes: [bold]{profile.duplicate_row_count}[/]",
+            title="Dataset Overview",
+        )
+    )
+
+    table = Table(title="Column Profiles")
+    table.add_column("Column", style="cyan")
+    table.add_column("Type")
+    table.add_column("Nulls", justify="right")
+    table.add_column("Unique", justify="right")
+    table.add_column("Sample Values")
+
+    for name, col in profile.columns.items():
+        null_str = (
+            f"{col.null_count} ({col.null_ratio:.0%})" if col.null_count > 0 else "0"
+        )
+        samples = ", ".join(str(v) for v in col.sample_values[:3])
+        semantic = f" [{col.semantic_type}]" if col.semantic_type else ""
+        table.add_row(
+            name, f"{col.dtype}{semantic}", null_str, str(col.unique_count), samples
+        )
+
+    console.print(table)
+
+
+def format_issues(findings: list[Finding]) -> None:
+    """Print detected issues."""
+    if not findings:
+        console.print("[green]No issues detected![/]")
+        return
+
+    table = Table(title=f"Issues Found ({len(findings)})")
+    table.add_column("Severity", justify="center")
+    table.add_column("Detector")
+    table.add_column("Message")
+    table.add_column("Columns")
+
+    for f in findings:
+        color = SEVERITY_COLORS.get(f.severity.value, "white")
+        table.add_row(
+            Text(f.severity.value.upper(), style=color),
+            f.detector,
+            f.message,
+            ", ".join(f.columns),
+        )
+
+    console.print(table)
+
+
+def format_suggestions(suggestions: list[Suggestion]) -> None:
+    """Print suggestions."""
+    if not suggestions:
+        console.print("[green]No suggestions.[/]")
+        return
+
+    table = Table(title=f"Suggestions ({len(suggestions)})")
+    table.add_column("#", justify="right", style="bold")
+    table.add_column("Priority", justify="center")
+    table.add_column("Action")
+    table.add_column("Impact")
+    table.add_column("Rationale")
+
+    for s in suggestions:
+        color = SEVERITY_COLORS.get(s.priority.value, "white")
+        table.add_row(
+            str(s.id),
+            Text(s.priority.value.upper(), style=color),
+            s.action,
+            s.impact,
+            s.rationale,
+        )
+
+    console.print(table)
+
+
+def format_transformation_summary(
+    before_rows: int, after_rows: int, n_applied: int
+) -> None:
+    """Print transformation summary."""
+    console.print(
+        Panel(
+            f"Applied [bold]{n_applied}[/] transformation(s)\n"
+            f"Rows: {before_rows} → {after_rows} ({before_rows - after_rows} removed)",
+            title="Cleaning Summary",
+        )
+    )

datawash/cli/main.py

@@ -0,0 +1,168 @@
+"""CLI entry point for datawash."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+
+app = typer.Typer(
+    name="datawash",
+    help="Intelligent data cleaning and quality analysis.",
+    no_args_is_help=True,
+)
+console = Console()
+
+
+@app.command()
+def analyze(
+    file: Path = typer.Argument(..., help="Path to data file"),
+    sample: Optional[int] = typer.Option(
+        None, "--sample", "-s", help="Number of rows to sample"
+    ),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
+) -> None:
+    """Analyze a dataset and show quality report."""
+    from datawash.cli.formatters import (
+        format_issues,
+        format_profile,
+        format_suggestions,
+    )
+    from datawash.core.report import Report
+
+    with console.status("Analyzing..."):
+        report = Report(str(file))
+
+    format_profile(report.profile)
+    console.print()
+    format_issues(report.issues)
+    console.print()
+    format_suggestions(report.suggestions)
+
+
+@app.command()
+def suggest(
+    file: Path = typer.Argument(..., help="Path to data file"),
+    use_case: str = typer.Option(
+        "general", "--use-case", "-u", help="Context: general, ml, analytics, export"
+    ),
+    priority: str = typer.Option(
+        "all", "--priority", "-p", help="Filter: high, medium, low, all"
+    ),
+    limit: int = typer.Option(20, "--limit", "-l", help="Max suggestions"),
+) -> None:
+    """Show cleaning suggestions for a dataset."""
+    from datawash.cli.formatters import format_suggestions
+    from datawash.core.report import Report
+
+    with console.status("Analyzing..."):
+        report = Report(str(file), use_case=use_case)
+
+    suggestions = report.suggestions
+    if priority != "all":
+        suggestions = [s for s in suggestions if s.priority.value == priority]
+    suggestions = suggestions[:limit]
+
+    format_suggestions(suggestions)
+
+
+@app.command()
+def clean(
+    file: Path = typer.Argument(..., help="Path to data file"),
+    output: Path = typer.Option(..., "--output", "-o", help="Output file path"),
+    apply: Optional[str] = typer.Option(
+        None, "--apply", "-a", help="Suggestion IDs (comma-separated)"
+    ),
+    apply_all: bool = typer.Option(False, "--apply-all", help="Apply all suggestions"),
+    use_case: str = typer.Option("general", "--use-case", "-u", help="Context"),
+    codegen: Optional[Path] = typer.Option(
+        None, "--codegen", help="Also save generated Python code"
+    ),
+) -> None:
+    """Clean a dataset by applying suggestions."""
+    from datawash.adapters.base import _ADAPTERS
+    from datawash.cli.formatters import format_transformation_summary
+    from datawash.core.report import Report
+
+    with console.status("Analyzing..."):
+        report = Report(str(file), use_case=use_case)
+
+    before_rows = len(report.df)
+
+    if apply_all:
+        clean_df = report.apply_all()
+    elif apply:
+        ids = [int(x.strip()) for x in apply.split(",")]
+        clean_df = report.apply(ids)
+    else:
+        console.print("[red]Specify --apply or --apply-all[/]")
+        raise typer.Exit(1)
+
+    # Save output
+    ext = output.suffix.lstrip(".")
+    adapter = _ADAPTERS.get(ext)
+    if adapter is None:
+        console.print(f"[red]Unsupported output format: {ext}[/]")
+        raise typer.Exit(1)
+    adapter.write(clean_df, output)
+    console.print(f"Saved cleaned data to [bold]{output}[/]")
+
+    format_transformation_summary(before_rows, len(clean_df), len(report._applied))
+
+    # Show before/after quality score
+    score_before = getattr(report, "_last_score_before", None)
+    score_after = getattr(report, "_last_score_after", None)
+    if score_before is not None and score_after is not None:
+        diff = score_after - score_before
+        sign = "+" if diff >= 0 else ""
+        console.print(f"Quality score: {score_before} → {score_after} ({sign}{diff})")
+
+    if codegen:
+        code = report.generate_code()
+        codegen.write_text(code)
+        console.print(f"Saved code to [bold]{codegen}[/]")
+
+
+@app.command()
+def codegen(
+    file: Path = typer.Argument(..., help="Path to data file"),
+    apply: Optional[str] = typer.Option(
+        None, "--apply", "-a", help="Suggestion IDs (comma-separated)"
+    ),
+    apply_all: bool = typer.Option(
+        False, "--apply-all", help="Generate code for all suggestions"
+    ),
+    style: str = typer.Option(
+        "function", "--style", "-s", help="Code style: function or script"
+    ),
+    output: Optional[Path] = typer.Option(
+        None, "--output", "-o", help="Output Python file"
+    ),
+) -> None:
+    """Generate Python code for data cleaning transformations."""
+    from datawash.core.report import Report
+
+    with console.status("Analyzing..."):
+        report = Report(str(file))
+
+    if apply_all:
+        report.apply_all()
+    elif apply:
+        ids = [int(x.strip()) for x in apply.split(",")]
+        report.apply(ids)
+    else:
+        report.apply_all()
+
+    code = report.generate_code(style=style)
+
+    if output:
+        output.write_text(code)
+        console.print(f"Saved code to [bold]{output}[/]")
+    else:
+        console.print(code)
+
+
+if __name__ == "__main__":
+    app()

datawash/codegen/__init__.py

@@ -0,0 +1 @@
+from .generator import generate_code as generate_code

datawash/codegen/generator.py

@@ -0,0 +1,72 @@
+"""Python code generation from transformation log."""
+
+from __future__ import annotations
+
+from datawash.core.models import TransformationResult
+
+
+def generate_code(
+    results: list[TransformationResult],
+    style: str = "function",
+    include_comments: bool = True,
+) -> str:
+    """Generate Python code from a list of transformation results.
+
+    Args:
+        results: List of TransformationResult from applied transformations.
+        style: "function" wraps in a function, "script" generates standalone.
+        include_comments: Whether to add explanatory comments.
+
+    Returns:
+        Python source code as a string.
+    """
+    if not results:
+        return "# No transformations to apply"
+
+    lines: list[str] = []
+
+    # Header
+    lines.append("import pandas as pd")
+    lines.append("import numpy as np")
+    lines.append("")
+
+    if style == "function":
+        lines.append("")
+        lines.append("def clean_data(df: pd.DataFrame) -> pd.DataFrame:")
+        lines.append('    """Apply data cleaning transformations."""')
+        lines.append("    df = df.copy()")
+        lines.append("")
+        for result in results:
+            if include_comments:
+                lines.append(
+                    f"    # {result.transformer}: {result.rows_affected} rows affected"
+                )
+            for code_line in result.code.split("\n"):
+                if code_line.strip():
+                    # Skip redundant imports inside function
+                    if code_line.startswith("import "):
+                        continue
+                    lines.append(f"    {code_line}")
+            lines.append("")
+        lines.append("    return df")
+    else:
+        if include_comments:
+            lines.append("# Load data")
+        lines.append('df = pd.read_csv("input.csv")  # Update path as needed')
+        lines.append("")
+        for result in results:
+            if include_comments:
+                lines.append(
+                    f"# {result.transformer}: {result.rows_affected} rows affected"
+                )
+            for code_line in result.code.split("\n"):
+                if code_line.strip():
+                    if code_line.startswith("import "):
+                        continue
+                    lines.append(code_line)
+            lines.append("")
+        if include_comments:
+            lines.append("# Save cleaned data")
+        lines.append('df.to_csv("output.csv", index=False)')
+
+    return "\n".join(lines) + "\n"

datawash/core/__init__.py

@@ -0,0 +1 @@
+"""Core functionality for datawash."""

datawash/core/cache.py

@@ -0,0 +1,64 @@
+"""Lazy-evaluated computation cache shared across detectors."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+
+class ComputationCache:
+    """Cache expensive column computations.
+
+    Computes on first access, returns cached value on subsequent access.
+    All detectors share the same cache instance to avoid redundant work.
+    """
+
+    def __init__(self, df: pd.DataFrame) -> None:
+        self._df = df
+        self._null_masks: dict[str, pd.Series] = {}
+        self._value_sets: dict[str, set[str]] = {}
+        self._unique_counts: dict[str, int] = {}
+        self._statistics: dict[str, dict[str, Any]] = {}
+
+    def get_null_mask(self, column: str) -> pd.Series:
+        """Return boolean mask of null values. Cached."""
+        if column not in self._null_masks:
+            self._null_masks[column] = self._df[column].isna()
+        return self._null_masks[column]
+
+    def get_value_set(self, column: str, max_values: int = 10000) -> set[str]:
+        """Return set of unique non-null string values. Cached."""
+        if column not in self._value_sets:
+            values = self._df[column].dropna()
+            if len(values) > max_values:
+                values = values.sample(max_values, random_state=42)
+            self._value_sets[column] = set(values.astype(str))
+        return self._value_sets[column]
+
+    def get_unique_count(self, column: str) -> int:
+        """Return count of unique values. Cached."""
+        if column not in self._unique_counts:
+            self._unique_counts[column] = int(self._df[column].nunique())
+        return self._unique_counts[column]
+
+    def get_statistics(self, column: str) -> dict[str, Any]:
+        """Return numeric statistics. Cached."""
+        if column not in self._statistics:
+            col = self._df[column]
+            if pd.api.types.is_numeric_dtype(col):
+                clean = col.dropna()
+                if clean.empty:
+                    self._statistics[column] = {}
+                else:
+                    self._statistics[column] = {
+                        "mean": float(clean.mean()),
+                        "std": float(clean.std()) if len(clean) > 1 else 0.0,
+                        "min": float(clean.min()),
+                        "max": float(clean.max()),
+                        "q1": float(clean.quantile(0.25)),
+                        "q3": float(clean.quantile(0.75)),
+                    }
+            else:
+                self._statistics[column] = {}
+        return self._statistics[column]

datawash/core/config.py

@@ -0,0 +1,56 @@
+"""Configuration management for datawash."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class DetectorConfig(BaseModel):
+    enabled: list[str] = Field(
+        default_factory=lambda: [
+            "missing",
+            "duplicates",
+            "types",
+            "formats",
+            "outliers",
+            "similarity",
+        ]
+    )
+    similarity_name_threshold: float = 0.8
+    similarity_value_threshold: float = 0.7
+    outlier_method: Literal["iqr", "zscore"] = "iqr"
+    outlier_threshold: float = 1.5
+    fuzzy_duplicates_enabled: bool = False
+    fuzzy_duplicates_threshold: float = 0.85
+
+
+class MLConfig(BaseModel):
+    embedding_model: str = "all-MiniLM-L6-v2"
+    device: str = "cpu"
+
+
+class SuggestionConfig(BaseModel):
+    max_suggestions: int = 50
+    min_confidence: float = 0.7
+
+
+class CodegenConfig(BaseModel):
+    style: Literal["function", "script"] = "function"
+    include_comments: bool = True
+
+
+class Config(BaseModel):
+    sample_size: int = 10000
+    max_unique_ratio: float = 0.95
+    null_threshold: float = 0.5
+    detectors: DetectorConfig = Field(default_factory=DetectorConfig)
+    ml: MLConfig = Field(default_factory=MLConfig)
+    suggestions: SuggestionConfig = Field(default_factory=SuggestionConfig)
+    codegen: CodegenConfig = Field(default_factory=CodegenConfig)
+    use_case: Literal["general", "ml", "analytics", "export"] = "general"
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Config:
+        return cls.model_validate(data)