datawash 0.2.0__py3-none-any.whl

datawash/detectors/duplicate_detector.py

@@ -0,0 +1,56 @@
+"""Detect duplicate rows."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from datawash.core.models import DatasetProfile, Finding, Severity
+from datawash.detectors.base import BaseDetector
+from datawash.detectors.registry import register_detector
+
+
+class DuplicateDetector(BaseDetector):
+    @property
+    def name(self) -> str:
+        return "duplicates"
+
+    @property
+    def description(self) -> str:
+        return "Detects exact duplicate rows"
+
+    def detect(self, df: pd.DataFrame, profile: DatasetProfile) -> list[Finding]:
+        findings: list[Finding] = []
+        dup_count = profile.duplicate_row_count
+        if dup_count == 0:
+            return findings
+
+        ratio = dup_count / profile.row_count if profile.row_count > 0 else 0
+        if ratio > 0.1:
+            severity = Severity.HIGH
+        elif ratio > 0.01:
+            severity = Severity.MEDIUM
+        else:
+            severity = Severity.LOW
+
+        dup_mask = df.duplicated(keep="first")
+        dup_indices = df.index[dup_mask].tolist()[:100]  # Cap for memory
+
+        findings.append(
+            Finding(
+                detector=self.name,
+                issue_type="duplicate_rows",
+                severity=severity,
+                columns=list(df.columns),
+                rows=dup_indices,
+                details={
+                    "duplicate_count": dup_count,
+                    "duplicate_ratio": round(ratio, 4),
+                },
+                message=f"Found {dup_count} exact duplicate rows ({ratio:.1%} of data)",
+                confidence=1.0,
+            )
+        )
+        return findings
+
+
+register_detector(DuplicateDetector())

datawash/detectors/format_detector.py

@@ -0,0 +1,130 @@
+"""Detect inconsistent formats within columns."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from datawash.core.models import DatasetProfile, Finding, Severity
+from datawash.detectors.base import BaseDetector
+from datawash.detectors.registry import register_detector
+
+
+class FormatDetector(BaseDetector):
+    @property
+    def name(self) -> str:
+        return "formats"
+
+    @property
+    def description(self) -> str:
+        return "Detects inconsistent formats within columns"
+
+    def detect(self, df: pd.DataFrame, profile: DatasetProfile) -> list[Finding]:
+        findings: list[Finding] = []
+        for col_name in df.columns:
+            if not pd.api.types.is_string_dtype(df[col_name]):
+                continue
+            clean = df[col_name].dropna().astype(str)
+            if clean.empty or len(clean) < 5:
+                continue
+
+            # Check for mixed case patterns
+            case_finding = self._check_case_inconsistency(col_name, clean)
+            if case_finding:
+                findings.append(case_finding)
+
+            # Check for mixed date formats
+            date_finding = self._check_date_formats(col_name, clean)
+            if date_finding:
+                findings.append(date_finding)
+
+            # Check for mixed whitespace/padding
+            ws_finding = self._check_whitespace(col_name, clean)
+            if ws_finding:
+                findings.append(ws_finding)
+
+        return findings
+
+    def _check_case_inconsistency(
+        self, col_name: str, series: pd.Series
+    ) -> Finding | None:
+        has_upper = series.str.isupper().any()
+        has_lower = series.str.islower().any()
+        has_title = series.str.istitle().any()
+        case_types = sum([has_upper, has_lower, has_title])
+        if case_types >= 2:
+            return Finding(
+                detector=self.name,
+                issue_type="inconsistent_case",
+                severity=Severity.LOW,
+                columns=[col_name],
+                details={
+                    "has_upper": bool(has_upper),
+                    "has_lower": bool(has_lower),
+                    "has_title": bool(has_title),
+                },
+                message=(
+                    f"Column '{col_name}' has inconsistent "
+                    f"casing (mixed upper/lower/title case)"
+                ),
+                confidence=0.8,
+            )
+        return None
+
+    _DATE_PATTERNS: list[tuple[str, str]] = [
+        ("iso", r"^\d{4}-\d{2}-\d{2}"),
+        ("slash_mdy", r"^\d{1,2}/\d{1,2}/\d{2,4}$"),
+        ("dash_dmy", r"^\d{1,2}-[A-Za-z]{3}-\d{2,4}$"),
+        ("named_mdy", r"^[A-Za-z]{3,9}\s+\d{1,2},?\s+\d{4}$"),
+        ("named_dmy", r"^\d{1,2}\s+[A-Za-z]{3,9}\s+\d{4}$"),
+    ]
+
+    def _check_date_formats(self, col_name: str, series: pd.Series) -> Finding | None:
+        # Count how many distinct format patterns appear
+        format_counts: dict[str, int] = {}
+        for fmt_name, pattern in self._DATE_PATTERNS:
+            count = int(series.str.match(pattern).sum())
+            if count > 0:
+                format_counts[fmt_name] = count
+
+        total_matched = sum(format_counts.values())
+        if len(format_counts) >= 2 and total_matched / len(series) >= 0.5:
+            detail_str = ", ".join(
+                f"{count} {name}" for name, count in format_counts.items()
+            )
+            return Finding(
+                detector=self.name,
+                issue_type="inconsistent_date_format",
+                severity=Severity.MEDIUM,
+                columns=[col_name],
+                details={
+                    "format_counts": format_counts,
+                    "total_matched": total_matched,
+                },
+                message=(
+                    f"Column '{col_name}' has mixed date " f"formats ({detail_str})"
+                ),
+                confidence=0.85,
+            )
+        return None
+
+    def _check_whitespace(self, col_name: str, series: pd.Series) -> Finding | None:
+        leading = (series != series.str.lstrip()).sum()
+        trailing = (series != series.str.rstrip()).sum()
+        total = int(leading + trailing)
+        if total > 0:
+            return Finding(
+                detector=self.name,
+                issue_type="whitespace_padding",
+                severity=Severity.LOW,
+                columns=[col_name],
+                details={
+                    "leading_spaces": int(leading),
+                    "trailing_spaces": int(trailing),
+                },
+                message=f"Column '{col_name}' has {total} values with leading/trailing whitespace",
+                confidence=1.0,
+            )
+        return None
+
+
+register_detector(FormatDetector())

datawash/detectors/missing_detector.py

@@ -0,0 +1,78 @@
+"""Detect missing value patterns."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from datawash.core.models import DatasetProfile, Finding, Severity
+from datawash.detectors.base import BaseDetector
+from datawash.detectors.registry import register_detector
+
+
+class MissingDetector(BaseDetector):
+    @property
+    def name(self) -> str:
+        return "missing"
+
+    @property
+    def description(self) -> str:
+        return "Detects missing values and null patterns"
+
+    def detect(self, df: pd.DataFrame, profile: DatasetProfile) -> list[Finding]:
+        findings: list[Finding] = []
+        for col_name, col_profile in profile.columns.items():
+            if col_profile.null_count == 0:
+                continue
+            ratio = col_profile.null_ratio
+            if ratio > 0.5:
+                severity = Severity.HIGH
+            elif ratio > 0.1:
+                severity = Severity.MEDIUM
+            else:
+                severity = Severity.LOW
+
+            findings.append(
+                Finding(
+                    detector=self.name,
+                    issue_type="missing_values",
+                    severity=severity,
+                    columns=[col_name],
+                    details={
+                        "null_count": col_profile.null_count,
+                        "null_ratio": col_profile.null_ratio,
+                        "dtype": col_profile.dtype,
+                    },
+                    message=(
+                        f"Column '{col_name}' has "
+                        f"{col_profile.null_count} missing "
+                        f"values ({col_profile.null_ratio:.1%})"
+                    ),
+                    confidence=1.0,
+                )
+            )
+
+        # Detect columns with empty or whitespace-only strings
+        for col_name in df.columns:
+            if pd.api.types.is_string_dtype(df[col_name]):
+                stripped = df[col_name].dropna().astype(str).str.strip()
+                empty_count = int((stripped == "").sum())
+                if empty_count > 0:
+                    findings.append(
+                        Finding(
+                            detector=self.name,
+                            issue_type="empty_strings",
+                            severity=Severity.MEDIUM,
+                            columns=[col_name],
+                            details={"empty_string_count": empty_count},
+                            message=(
+                                f"Column '{col_name}' has "
+                                f"{empty_count} empty strings "
+                                f"that may represent missing values"
+                            ),
+                            confidence=0.9,
+                        )
+                    )
+        return findings
+
+
+register_detector(MissingDetector())

datawash/detectors/outlier_detector.py

@@ -0,0 +1,93 @@
+"""Detect statistical outliers in numeric columns."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from datawash.core.models import DatasetProfile, Finding, Severity
+from datawash.detectors.base import BaseDetector
+from datawash.detectors.registry import register_detector
+
+
+class OutlierDetector(BaseDetector):
+    def __init__(self, method: str = "iqr", threshold: float = 1.5) -> None:
+        self._method = method
+        self._threshold = threshold
+
+    @property
+    def name(self) -> str:
+        return "outliers"
+
+    @property
+    def description(self) -> str:
+        return "Detects statistical outliers in numeric columns"
+
+    def detect(self, df: pd.DataFrame, profile: DatasetProfile) -> list[Finding]:
+        findings: list[Finding] = []
+        for col_name in df.select_dtypes(include=[np.number]).columns:
+            series = df[col_name].dropna()
+            if len(series) < 10:
+                continue
+
+            if self._method == "iqr":
+                outlier_indices = self._iqr_outliers(series)
+            else:
+                outlier_indices = self._zscore_outliers(series)
+
+            if len(outlier_indices) == 0:
+                continue
+
+            ratio = len(outlier_indices) / len(series)
+            severity = (
+                Severity.HIGH
+                if ratio > 0.05
+                else Severity.MEDIUM if ratio > 0.01 else Severity.LOW
+            )
+
+            findings.append(
+                Finding(
+                    detector=self.name,
+                    issue_type="outliers",
+                    severity=severity,
+                    columns=[col_name],
+                    rows=outlier_indices[:100],
+                    details={
+                        "outlier_count": len(outlier_indices),
+                        "outlier_ratio": round(ratio, 4),
+                        "method": self._method,
+                        "threshold": self._threshold,
+                    },
+                    message=(
+                        f"Column '{col_name}' has "
+                        f"{len(outlier_indices)} outliers "
+                        f"({ratio:.1%}) detected by "
+                        f"{self._method.upper()}"
+                    ),
+                    confidence=0.85,
+                )
+            )
+        return findings
+
+    def _iqr_outliers(self, series: pd.Series) -> list[int]:
+        q1 = series.quantile(0.25)
+        q3 = series.quantile(0.75)
+        iqr = q3 - q1
+        if iqr == 0:
+            return []
+        lower = q1 - self._threshold * iqr
+        upper = q3 + self._threshold * iqr
+        mask = (series < lower) | (series > upper)
+        return series.index[mask].tolist()
+
+    def _zscore_outliers(self, series: pd.Series) -> list[int]:
+        mean = series.mean()
+        std = series.std()
+        if std == 0:
+            return []
+        z_scores = ((series - mean) / std).abs()
+        mask = z_scores > self._threshold
+        return series.index[mask].tolist()
+
+
+register_detector(OutlierDetector())

datawash/detectors/registry.py

@@ -0,0 +1,64 @@
+"""Detector registration and orchestration."""
+
+from __future__ import annotations
+
+import logging
+import sys
+from typing import Optional
+
+import pandas as pd
+
+from datawash.core.models import DatasetProfile, Finding
+from datawash.detectors.base import BaseDetector
+
+logger = logging.getLogger(__name__)
+
+_DETECTORS: dict[str, BaseDetector] = {}
+
+
+def register_detector(detector: BaseDetector) -> None:
+    _DETECTORS[detector.name] = detector
+
+
+def get_all_detectors() -> dict[str, BaseDetector]:
+    return dict(_DETECTORS)
+
+
+def run_all_detectors(
+    df: pd.DataFrame,
+    profile: DatasetProfile,
+    enabled: Optional[list[str]] = None,
+) -> list[Finding]:
+    """Run enabled detectors and return all findings."""
+    findings: list[Finding] = []
+    active_detectors = {
+        n: d for n, d in _DETECTORS.items() if enabled is None or n in enabled
+    }
+    use_progress = len(df) > 10000 and sys.stderr.isatty()
+
+    if use_progress:
+        from rich.progress import Progress
+
+        with Progress() as progress:
+            task = progress.add_task(
+                "Running detectors...", total=len(active_detectors)
+            )
+            for name, detector in active_detectors.items():
+                try:
+                    logger.info("Running detector: %s", name)
+                    results = detector.detect(df, profile)
+                    findings.extend(results)
+                    logger.info("Detector %s found %d issues", name, len(results))
+                except Exception:
+                    logger.exception("Detector %s failed", name)
+                progress.update(task, advance=1)
+    else:
+        for name, detector in active_detectors.items():
+            try:
+                logger.info("Running detector: %s", name)
+                results = detector.detect(df, profile)
+                findings.extend(results)
+                logger.info("Detector %s found %d issues", name, len(results))
+            except Exception:
+                logger.exception("Detector %s failed", name)
+    return findings