dataruff 0.1.0__py3-none-any.whl

datadoctor/__init__.py

@@ -0,0 +1,36 @@
+"""
+dataruff — One-command dataset health diagnostics.
+
+Usage:
+    from datadoctor import audit, fix, score, detect_pii
+
+    audit(df)                    # Print quality report
+    fix(df)                      # Return cleaned DataFrame
+    score(df)                    # Return ScoreBreakdown
+    detect_pii(df)               # Return PIIReport
+"""
+
+from datadoctor.audit import audit
+from datadoctor.investigate import investigate
+from datadoctor.fix import fix
+from datadoctor.validate import validate
+from datadoctor.compare import compare
+from datadoctor.pii import detect_pii, mask_pii
+from datadoctor.drift import detect_drift
+from datadoctor.anomalies import find_anomalies
+from datadoctor.score import score
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "audit",
+    "investigate",
+    "fix",
+    "validate",
+    "compare",
+    "detect_pii",
+    "mask_pii",
+    "detect_drift",
+    "find_anomalies",
+    "score",
+]

datadoctor/_compat.py

@@ -0,0 +1,40 @@
+"""
+Pandas 2.x / 3.x dtype compatibility.
+
+In pandas 2.x string columns have dtype ``object`` (dtype.name == 'object').
+In pandas 3.x (infer_string=True by default) they have a ``StringDtype``
+instance whose repr shows as ``dtype: str`` (dtype.name may be 'str',
+'string', or 'string[python]' depending on the sub-release).
+
+The safest guard is ``isinstance(dtype, pd.StringDtype)`` — it covers every
+StringDtype variant without relying on the `.name` attribute.
+"""
+from __future__ import annotations
+
+import pandas as pd
+
+
+def is_str_col(series: pd.Series) -> bool:
+    """
+    True for any string-like column, pandas 2.x and 3.x compatible.
+
+    - pandas 2.x default:  dtype == object   (plain Python objects)
+    - pandas 3.x default:  isinstance(dtype, pd.StringDtype)
+                            (repr shows as ``dtype: str``)
+    """
+    dtype = series.dtype
+    # Fast path: classic object dtype used by pandas 2.x
+    if dtype == object:
+        return True
+    # All StringDtype variants (pd.StringDtype was added in pandas 1.0 and is
+    # the default in pandas 3.x regardless of storage backend)
+    if hasattr(pd, "StringDtype") and isinstance(dtype, pd.StringDtype):
+        return True
+    # Extra safety-net: catch any future/vendor string dtype by name
+    name = getattr(dtype, "name", "")
+    return name in ("str", "string", "large_string") or "string" in str(dtype).lower()
+
+
+def str_columns(df: pd.DataFrame) -> list[str]:
+    """Return names of all string-like columns in *df*."""
+    return [col for col in df.columns if is_str_col(df[col])]

datadoctor/analyzers/__init__.py

@@ -0,0 +1,19 @@
+from datadoctor.analyzers import (
+    duplicate,
+    null_analyzer,
+    type_analyzer,
+    format_analyzer,
+    outlier,
+    pii_analyzer,
+    drift_analyzer,
+)
+
+__all__ = [
+    "duplicate",
+    "null_analyzer",
+    "type_analyzer",
+    "format_analyzer",
+    "outlier",
+    "pii_analyzer",
+    "drift_analyzer",
+]

datadoctor/analyzers/drift_analyzer.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from datadoctor._compat import is_str_col
+
+_KS_SIGNIFICANCE = 0.05
+_CATEGORY_DRIFT_THRESHOLD = 0.05
+
+
+def analyze(old_df: pd.DataFrame, new_df: pd.DataFrame) -> dict[str, Any]:
+    distribution_drift: dict[str, float] = {}
+    category_drift: dict[str, dict[str, Any]] = {}
+    missing_value_drift: dict[str, float] = {}
+    drifted: set[str] = set()
+
+    common_cols = set(old_df.columns) & set(new_df.columns)
+
+    for col in sorted(common_cols):
+        old_s = old_df[col]
+        new_s = new_df[col]
+
+        # Missing-value drift
+        old_null = old_s.isna().mean()
+        new_null = new_s.isna().mean()
+        mv_change = round(abs(new_null - old_null) * 100, 2)
+        missing_value_drift[col] = mv_change
+        if mv_change > 5.0:
+            drifted.add(col)
+
+        # Numeric distribution drift (KS test)
+        if pd.api.types.is_numeric_dtype(old_s) and pd.api.types.is_numeric_dtype(new_s):
+            old_clean = old_s.dropna().astype(float)
+            new_clean = new_s.dropna().astype(float)
+            if len(old_clean) > 1 and len(new_clean) > 1:
+                stat, p_value = stats.ks_2samp(old_clean, new_clean)
+                distribution_drift[col] = round(float(stat), 4)
+                if p_value < _KS_SIGNIFICANCE:
+                    drifted.add(col)
+
+        # Categorical distribution drift
+        elif is_str_col(old_s) and is_str_col(new_s):
+            old_freq = old_s.value_counts(normalize=True)
+            new_freq = new_s.value_counts(normalize=True)
+            all_cats = set(old_freq.index) | set(new_freq.index)
+            changes: dict[str, Any] = {}
+            for cat in all_cats:
+                old_p = float(old_freq.get(cat, 0.0))
+                new_p = float(new_freq.get(cat, 0.0))
+                if abs(new_p - old_p) > _CATEGORY_DRIFT_THRESHOLD:
+                    changes[str(cat)] = {
+                        "old_pct": round(old_p * 100, 2),
+                        "new_pct": round(new_p * 100, 2),
+                    }
+            if changes:
+                category_drift[col] = changes
+                drifted.add(col)
+
+    return {
+        "distribution_drift": distribution_drift,
+        "category_drift": category_drift,
+        "missing_value_drift": missing_value_drift,
+        "drifted_columns": sorted(drifted),
+    }

datadoctor/analyzers/duplicate.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import pandas as pd
+
+from datadoctor.models import Issue
+
+_HIGH_THRESHOLD = 0.10  # >10% duplicates → high severity
+
+
+def analyze(df: pd.DataFrame) -> list[Issue]:
+    if df.empty:
+        return []
+
+    mask = df.duplicated()
+    dup_count = int(mask.sum())
+
+    if dup_count == 0:
+        return []
+
+    pct = dup_count / len(df)
+    severity = "high" if pct > _HIGH_THRESHOLD else "medium"
+
+    return [
+        Issue(
+            type="duplicate_rows",
+            severity=severity,
+            count=dup_count,
+            details={
+                "percentage": round(pct * 100, 2),
+                "duplicate_indices": df[mask].index.tolist(),
+            },
+        )
+    ]

datadoctor/analyzers/format_analyzer.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+from dateutil.parser import ParserError
+from dateutil.parser import parse as parse_date
+
+from datadoctor._compat import is_str_col
+from datadoctor.models import Issue
+
+_EMAIL_RE = re.compile(r"^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$")
+_EMAIL_HINTS = ("email", "mail", "e-mail", "e_mail")
+_DATE_HINTS = ("date", "time", "dt", "created", "updated", "modified", "timestamp")
+
+
+def _is_email_col(name: str) -> bool:
+    low = name.lower()
+    return any(h in low for h in _EMAIL_HINTS)
+
+
+def _is_date_col(name: str) -> bool:
+    low = name.lower()
+    return any(h in low for h in _DATE_HINTS)
+
+
+def analyze(df: pd.DataFrame) -> list[Issue]:
+    issues: list[Issue] = []
+
+    for col in df.columns:
+        if not is_str_col(df[col]):
+            continue
+
+        series = df[col].dropna().astype(str)
+        if len(series) == 0:
+            continue
+
+        if _is_email_col(col):
+            invalid = series[~series.str.match(_EMAIL_RE)]
+            if len(invalid) > 0:
+                issues.append(
+                    Issue(
+                        type="invalid_email",
+                        severity="medium",
+                        count=len(invalid),
+                        column=col,
+                        details={"examples": invalid.head(3).tolist()},
+                    )
+                )
+
+        if _is_date_col(col):
+            formats_seen: set[str] = set()
+            parse_errors = 0
+
+            for val in series:
+                try:
+                    parse_date(val, fuzzy=False)
+                    if re.match(r"^\d{4}-\d{2}-\d{2}", val):
+                        formats_seen.add("YYYY-MM-DD")
+                    elif re.match(r"^\d{1,2}/\d{1,2}/\d{2,4}", val):
+                        formats_seen.add("MM/DD/YYYY")
+                    elif re.match(r"^\d{1,2}-\d{1,2}-\d{2,4}", val):
+                        formats_seen.add("DD-MM-YYYY")
+                    else:
+                        formats_seen.add("other")
+                except (ParserError, ValueError, OverflowError):
+                    parse_errors += 1
+
+            if parse_errors > 0:
+                issues.append(
+                    Issue(
+                        type="invalid_date",
+                        severity="medium",
+                        count=parse_errors,
+                        column=col,
+                        details={"unparseable_count": parse_errors},
+                    )
+                )
+
+            if len(formats_seen) > 1:
+                issues.append(
+                    Issue(
+                        type="inconsistent_date_format",
+                        severity="low",
+                        count=len(series),
+                        column=col,
+                        details={"formats_detected": sorted(formats_seen)},
+                    )
+                )
+
+    return issues

datadoctor/analyzers/null_analyzer.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import pandas as pd
+
+from datadoctor.models import Issue
+
+_HIGH_NULL_THRESHOLD = 0.30
+_MEDIUM_NULL_THRESHOLD = 0.05
+
+
+def analyze(df: pd.DataFrame) -> list[Issue]:
+    if df.empty:
+        return []
+
+    issues: list[Issue] = []
+
+    empty_cols = [col for col in df.columns if df[col].isna().all()]
+    if empty_cols:
+        issues.append(
+            Issue(
+                type="empty_columns",
+                severity="high",
+                count=len(empty_cols),
+                details={"columns": empty_cols},
+            )
+        )
+
+    for col in df.columns:
+        if col in empty_cols:
+            continue
+        null_count = int(df[col].isna().sum())
+        if null_count == 0:
+            continue
+        pct = null_count / len(df)
+        if pct > _HIGH_NULL_THRESHOLD:
+            severity = "high"
+        elif pct > _MEDIUM_NULL_THRESHOLD:
+            severity = "medium"
+        else:
+            severity = "low"
+
+        issues.append(
+            Issue(
+                type="null_values",
+                severity=severity,
+                count=null_count,
+                column=col,
+                details={"percentage": round(pct * 100, 2)},
+            )
+        )
+
+    return issues

datadoctor/analyzers/outlier.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from datadoctor.models import Issue
+
+_MIN_ROWS = 4
+_HIGH_OUTLIER_THRESHOLD = 0.10
+
+
+def analyze(df: pd.DataFrame, method: str = "iqr") -> list[Issue]:
+    issues: list[Issue] = []
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+
+    for col in numeric_cols:
+        series = df[col].dropna()
+        if len(series) < _MIN_ROWS:
+            continue
+
+        if method == "zscore":
+            mask = _zscore_mask(series)
+        else:
+            mask = _iqr_mask(series)
+
+        count = int(mask.sum())
+        if count == 0:
+            continue
+
+        pct = count / len(series)
+        severity = "high" if pct > _HIGH_OUTLIER_THRESHOLD else "medium"
+
+        issues.append(
+            Issue(
+                type="outlier",
+                severity=severity,
+                count=count,
+                column=col,
+                details={
+                    "method": method,
+                    "percentage": round(pct * 100, 2),
+                    "min": float(series.min()),
+                    "max": float(series.max()),
+                    "mean": round(float(series.mean()), 4),
+                    "std": round(float(series.std()), 4),
+                },
+            )
+        )
+
+    return issues
+
+
+def get_outlier_mask(series: pd.Series, method: str = "iqr") -> pd.Series:
+    if method == "zscore":
+        return _zscore_mask(series)
+    return _iqr_mask(series)
+
+
+def _iqr_mask(series: pd.Series) -> pd.Series:
+    q1 = series.quantile(0.25)
+    q3 = series.quantile(0.75)
+    iqr = q3 - q1
+    lower = q1 - 1.5 * iqr
+    upper = q3 + 1.5 * iqr
+    return (series < lower) | (series > upper)
+
+
+def _zscore_mask(series: pd.Series, threshold: float = 3.0) -> pd.Series:
+    std = series.std()
+    if std == 0:
+        return pd.Series(False, index=series.index)
+    z = (series - series.mean()) / std
+    return z.abs() > threshold

datadoctor/analyzers/pii_analyzer.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from datadoctor._compat import is_str_col
+
+_PATTERNS: dict[str, re.Pattern[str]] = {
+    "email": re.compile(
+        r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b"
+    ),
+    "phone": re.compile(
+        r"\b(?:\+?91[-.\s]?)?[6-9]\d{9}\b"
+        r"|\b\+?1?[-.\s]?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b"
+    ),
+    "aadhaar": re.compile(r"\b[2-9]\d{3}[\s-]?\d{4}[\s-]?\d{4}\b"),
+    "pan": re.compile(r"\b[A-Z]{5}[0-9]{4}[A-Z]\b"),
+    "ssn": re.compile(r"\b\d{3}-\d{2}-\d{4}\b"),
+    "credit_card": re.compile(
+        r"\b(?:4\d{3}|5[1-5]\d{2}|6011|3[47]\d{2})[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{4}\b"
+    ),
+}
+
+_COLUMN_HINTS: dict[str, list[str]] = {
+    "email": ["email", "mail", "e-mail", "e_mail"],
+    "phone": ["phone", "mobile", "cell", "tel", "contact", "phoneno", "phone_no"],
+    "aadhaar": ["aadhaar", "aadhar", "uid", "uidai"],
+    "pan": ["pan", "pan_number", "panno", "pan_no"],
+    "ssn": ["ssn", "social_security", "social_security_number"],
+    "credit_card": ["card", "cc_number", "credit_card", "creditcard", "cardno"],
+}
+
+_CONTENT_SCAN_SAMPLE = 200
+
+
+def analyze(df: pd.DataFrame) -> dict[str, list[str]]:
+    results: dict[str, list[str]] = {}
+
+    for col in df.columns:
+        pii_found: set[str] = set()
+        col_lower = col.lower()
+
+        for pii_type, hints in _COLUMN_HINTS.items():
+            if any(h in col_lower for h in hints):
+                pii_found.add(pii_type)
+
+        if is_str_col(df[col]):
+            sample = df[col].dropna().astype(str).head(_CONTENT_SCAN_SAMPLE)
+            for pii_type, pattern in _PATTERNS.items():
+                if pii_type not in pii_found:
+                    if any(pattern.search(val) for val in sample):
+                        pii_found.add(pii_type)
+
+        if pii_found:
+            results[col] = sorted(pii_found)
+
+    return results

datadoctor/analyzers/type_analyzer.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import pandas as pd
+
+from datadoctor._compat import is_str_col
+from datadoctor.models import Issue
+
+
+def analyze(df: pd.DataFrame) -> list[Issue]:
+    issues: list[Issue] = []
+
+    for col in df.columns:
+        if not is_str_col(df[col]):
+            continue
+
+        series = df[col].dropna()
+        if len(series) == 0:
+            continue
+
+        # Detect mixed Python types (e.g., int and str coexisting)
+        type_counts: dict[str, int] = {}
+        for val in series:
+            t = type(val).__name__
+            type_counts[t] = type_counts.get(t, 0) + 1
+
+        if len(type_counts) > 1 and not set(type_counts.keys()) <= {"str", "bytes"}:
+            issues.append(
+                Issue(
+                    type="mixed_types",
+                    severity="medium",
+                    count=len(series),
+                    column=col,
+                    details={"types_found": list(type_counts.keys())},
+                )
+            )
+            continue  # skip numeric-as-string check for mixed columns
+
+        # Detect numeric values stored as strings
+        numeric_mask = pd.to_numeric(series.astype(str).str.replace(",", "", regex=False), errors="coerce").notna()
+        if numeric_mask.all() and len(series) > 0:
+            issues.append(
+                Issue(
+                    type="numeric_as_string",
+                    severity="low",
+                    count=len(series),
+                    column=col,
+                    details={"suggestion": f"Column '{col}' contains numeric values stored as strings."},
+                )
+            )
+
+    return issues

datadoctor/anomalies.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Union
+
+import numpy as np
+import pandas as pd
+
+from datadoctor.analyzers.outlier import analyze as _outlier_analyze
+from datadoctor.analyzers.outlier import get_outlier_mask
+from datadoctor.loader import load
+
+
+def find_anomalies(
+    source: Union[str, Path, pd.DataFrame],
+    method: str = "iqr",
+) -> dict[str, Any]:
+    if method not in ("iqr", "zscore"):
+        raise ValueError(f"Unknown method '{method}'. Choose 'iqr' or 'zscore'.")
+
+    df = load(source)
+    issues = _outlier_analyze(df, method=method)
+
+    total = sum(i.count for i in issues)
+
+    # Collect the union of anomalous row indices
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    anomalous_mask = pd.Series(False, index=df.index)
+    for col in numeric_cols:
+        series = df[col].dropna()
+        if len(series) >= 4:
+            col_mask = get_outlier_mask(series, method=method)
+            full_mask = col_mask.reindex(df.index, fill_value=False)
+            anomalous_mask = anomalous_mask | full_mask
+
+    return {
+        "total_anomalous_records": int(anomalous_mask.sum()),
+        "method": method,
+        "by_column": [
+            {
+                "column": i.column,
+                "count": i.count,
+                "percentage": i.details.get("percentage"),
+                "severity": i.severity,
+                "min": i.details.get("min"),
+                "max": i.details.get("max"),
+                "mean": i.details.get("mean"),
+            }
+            for i in issues
+        ],
+        "anomalous_indices": anomalous_mask[anomalous_mask].index.tolist(),
+    }

datadoctor/audit.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from typing import Union
+from pathlib import Path
+
+import pandas as pd
+
+from datadoctor.investigate import investigate
+from datadoctor.models import InvestigationReport
+from datadoctor.reporting.terminal import print_audit_report
+
+
+def audit(
+    source: Union[str, Path, pd.DataFrame],
+    schema: dict | None = None,
+) -> InvestigationReport:
+    report = investigate(source, schema=schema)
+    print_audit_report(report)
+    return report