dataforge-07 0.1.0__py3-none-any.whl

dataforge/datasets/registry.py

@@ -0,0 +1,103 @@
+"""Canonical metadata for real-world benchmark datasets."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+
+class HeaderMismatch(BaseModel):
+    """Pair of dirty/clean header names that align by column position."""
+
+    dirty_name: str = Field(min_length=1)
+    clean_name: str = Field(min_length=1)
+
+    model_config = {"frozen": True}
+
+
+class DatasetMetadata(BaseModel):
+    """Metadata describing a canonical benchmark dataset."""
+
+    name: str = Field(min_length=1)
+    domain: str = Field(min_length=1)
+    n_rows: int = Field(ge=0)
+    n_columns: int = Field(ge=1)
+    error_types: tuple[str, ...] = Field(default_factory=tuple)
+    source_urls: tuple[str, str]
+    source_revision: str = Field(min_length=7)
+    dirty_sha256: str = Field(min_length=64, max_length=64)
+    clean_sha256: str = Field(min_length=64, max_length=64)
+    citation: str = Field(min_length=1)
+    header_mismatches: tuple[HeaderMismatch, ...] = Field(default_factory=tuple)
+
+    model_config = {"frozen": True}
+
+
+RAHA_GIT_REVISION = "7be1334b8c7bbdac3f47ef514fb3e1e8c5fc181c"
+_BASE_URL = f"https://raw.githubusercontent.com/BigDaMa/raha/{RAHA_GIT_REVISION}/datasets"
+
+DATASET_REGISTRY: dict[str, DatasetMetadata] = {
+    "hospital": DatasetMetadata(
+        name="hospital",
+        domain="healthcare",
+        n_rows=1000,
+        n_columns=20,
+        error_types=("typo", "missing_value", "formatting"),
+        source_urls=(
+            f"{_BASE_URL}/hospital/dirty.csv",
+            f"{_BASE_URL}/hospital/clean.csv",
+        ),
+        source_revision=RAHA_GIT_REVISION,
+        dirty_sha256="dbc5575b915fe8b5e0ac6dc6172f38ba91e611fdb76d09a8f4a81cb7ea9925ac",
+        clean_sha256="ea3ee44998455c0b491750c348509de176c758a3bbf58e4530c0a136bb248b4b",
+        citation=(
+            "Mahdavi et al. Raha benchmark dataset (Hospital) via the BigDaMa/raha repository."
+        ),
+    ),
+    "flights": DatasetMetadata(
+        name="flights",
+        domain="aviation",
+        n_rows=2376,
+        n_columns=7,
+        error_types=("missing_value", "formatting", "datetime"),
+        source_urls=(
+            f"{_BASE_URL}/flights/dirty.csv",
+            f"{_BASE_URL}/flights/clean.csv",
+        ),
+        source_revision=RAHA_GIT_REVISION,
+        dirty_sha256="1b5c1afa10aa0e7c20fd7e14d05c56772715b2771aa0f5fa67ed1709e1eecd46",
+        clean_sha256="0acfcfd8985b06fdd363965c9e8d9522c43e7589a93d79ae7dc311e1c37fdf3b",
+        citation=(
+            "Mahdavi et al. Raha benchmark dataset (Flights) via the BigDaMa/raha repository."
+        ),
+    ),
+    "beers": DatasetMetadata(
+        name="beers",
+        domain="consumer",
+        n_rows=2410,
+        n_columns=11,
+        error_types=("formatting", "missing_value", "normalization"),
+        source_urls=(
+            f"{_BASE_URL}/beers/dirty.csv",
+            f"{_BASE_URL}/beers/clean.csv",
+        ),
+        source_revision=RAHA_GIT_REVISION,
+        dirty_sha256="7110bf4931a9445a1675e544d6c996817c739136239f8a2b02e088c7ec0a1f68",
+        clean_sha256="373227df59ad197e154dd5149125789e415019535c7223355e9486ee1b3b93de",
+        citation=("Mahdavi et al. Raha benchmark dataset (Beers) via the BigDaMa/raha repository."),
+    ),
+}
+
+
+def get_dataset_metadata(name: str) -> DatasetMetadata:
+    """Return canonical metadata for a named benchmark dataset.
+
+    Args:
+        name: Canonical dataset name.
+
+    Returns:
+        The immutable metadata entry for the dataset.
+
+    Raises:
+        KeyError: If the dataset is not registered.
+    """
+    return DATASET_REGISTRY[name]

dataforge/detectors/__init__.py

@@ -0,0 +1,80 @@
+"""DataForge detector package — pure data-quality issue detection.
+
+This package provides the detector infrastructure and three Week 1 detectors:
+
+- :class:`TypeMismatchDetector` — numeric/string/date type conflicts.
+- :class:`DecimalShiftDetector` — power-of-10 outliers in numeric columns.
+- :class:`FDViolationDetector` — rows violating declared functional dependencies.
+
+Use :func:`run_all_detectors` to run all detectors and get a merged,
+deduplicated, severity-sorted issue list.
+"""
+
+from __future__ import annotations
+
+from dataforge.detectors.base import Detector, Issue, Schema, Severity
+from dataforge.detectors.decimal_shift import DecimalShiftDetector
+from dataforge.detectors.fd_violation import FDViolationDetector
+from dataforge.detectors.type_mismatch import TypeMismatchDetector
+from dataforge.table import TableLike
+
+__all__ = [
+    "DecimalShiftDetector",
+    "FDViolationDetector",
+    "Issue",
+    "Schema",
+    "Severity",
+    "TypeMismatchDetector",
+    "run_all_detectors",
+]
+
+# Severity sort key: UNSAFE first, then REVIEW, then SAFE.
+_SEVERITY_ORDER = {Severity.UNSAFE: 0, Severity.REVIEW: 1, Severity.SAFE: 2}
+
+
+def run_all_detectors(df: TableLike, schema: Schema | None = None) -> list[Issue]:
+    """Run all registered detectors and return a merged, sorted issue list.
+
+    Issues are deduplicated by (row, column, issue_type) and sorted by
+    severity (UNSAFE first) then confidence (highest first).
+
+    Args:
+        df: The input table to analyze.
+        schema: Optional declared schema with column types and constraints.
+
+    Returns:
+        A list of Issue objects from all detectors, sorted by severity
+        then confidence descending.
+
+    Example:
+        >>> import pandas as pd
+        >>> from dataforge.detectors import run_all_detectors
+        >>> df = pd.DataFrame({"age": ["25", "30", "N/A", "40"]})
+        >>> issues = run_all_detectors(df)
+        >>> len(issues)
+        1
+    """
+    detectors: list[Detector] = [
+        TypeMismatchDetector(),
+        DecimalShiftDetector(),
+        FDViolationDetector(),
+    ]
+
+    all_issues: list[Issue] = []
+    for detector in detectors:
+        all_issues.extend(detector.detect(df, schema))
+
+    # Deduplicate by (row, column, issue_type).
+    seen: set[tuple[int, str, str]] = set()
+    unique: list[Issue] = []
+    for issue in all_issues:
+        key = (issue.row, issue.column, issue.issue_type)
+        if key not in seen:
+            seen.add(key)
+            unique.append(issue)
+
+    # Sort: UNSAFE first, then REVIEW, then SAFE; within same severity,
+    # highest confidence first.
+    unique.sort(key=lambda i: (_SEVERITY_ORDER[i.severity], -i.confidence))
+
+    return unique

dataforge/detectors/base.py

@@ -0,0 +1,145 @@
+"""Core models and protocol for the DataForge detector subsystem."""
+
+from __future__ import annotations
+
+import enum
+from typing import Literal, Protocol
+
+from pydantic import BaseModel, Field
+
+from dataforge.table import TableLike
+from dataforge.verifier.schema import (
+    AcceptedValues,
+    AggregateDependency,
+    DomainBound,
+    FunctionalDependency,
+    RegexConstraint,
+    RelationshipConstraint,
+    Schema,
+)
+
+__all__ = [
+    "AggregateDependency",
+    "AcceptedValues",
+    "Detector",
+    "DomainBound",
+    "FunctionalDependency",
+    "Issue",
+    "IssueTypeLiteral",
+    "RegexConstraint",
+    "RelationshipConstraint",
+    "Schema",
+    "Severity",
+]
+
+
+class Severity(enum.Enum):
+    """Three-tier severity for data-quality issues.
+
+    Ordering: SAFE < REVIEW < UNSAFE (higher = more severe).
+
+    - SAFE: likely benign; can be auto-applied in bulk without human review.
+    - REVIEW: ambiguous; should appear in the profile table for human triage.
+    - UNSAFE: structural error; blocks automated repair without explicit approval.
+
+    See DECISIONS.md entry "Issue severity tiers" for the rationale behind
+    choosing exactly 3 levels.
+    """
+
+    SAFE = "safe"
+    REVIEW = "review"
+    UNSAFE = "unsafe"
+
+    def __lt__(self, other: object) -> bool:
+        """Enable ordering so SAFE < REVIEW < UNSAFE."""
+        if not isinstance(other, Severity):
+            return NotImplemented
+        order = {Severity.SAFE: 0, Severity.REVIEW: 1, Severity.UNSAFE: 2}
+        return order[self] < order[other]
+
+    def __le__(self, other: object) -> bool:
+        """Enable ordering so SAFE <= REVIEW <= UNSAFE."""
+        if not isinstance(other, Severity):
+            return NotImplemented
+        order = {Severity.SAFE: 0, Severity.REVIEW: 1, Severity.UNSAFE: 2}
+        return order[self] <= order[other]
+
+    def __gt__(self, other: object) -> bool:
+        """Enable ordering so UNSAFE > REVIEW > SAFE."""
+        if not isinstance(other, Severity):
+            return NotImplemented
+        order = {Severity.SAFE: 0, Severity.REVIEW: 1, Severity.UNSAFE: 2}
+        return order[self] > order[other]
+
+    def __ge__(self, other: object) -> bool:
+        """Enable ordering so UNSAFE >= REVIEW >= SAFE."""
+        if not isinstance(other, Severity):
+            return NotImplemented
+        order = {Severity.SAFE: 0, Severity.REVIEW: 1, Severity.UNSAFE: 2}
+        return order[self] >= order[other]
+
+
+# Closed vocabulary of issue types. Extend this Literal as new detectors ship.
+IssueTypeLiteral = Literal["type_mismatch", "decimal_shift", "fd_violation"]
+
+
+class Issue(BaseModel):
+    """A single data-quality finding at a specific (row, column) location.
+
+    Args:
+        row: Zero-indexed row number in the DataFrame.
+        column: Column name where the issue was detected.
+        issue_type: Machine-readable issue category (closed vocabulary).
+        severity: Three-tier severity classification.
+        confidence: Detector's confidence in the finding (0.0 to 1.0).
+        expected: What the value should be (if known); None for detection-only.
+        actual: The actual value found in the cell.
+        reason: Human-readable explanation of the issue.
+
+    Example:
+        >>> issue = Issue(
+        ...     row=3, column="price", issue_type="decimal_shift",
+        ...     severity=Severity.REVIEW, confidence=0.92,
+        ...     expected="102.0", actual="1020.0",
+        ...     reason="Value 1020.0 appears to be ~10x the typical value",
+        ... )
+    """
+
+    row: int = Field(ge=0, description="Zero-indexed row number")
+    column: str = Field(min_length=1, description="Column name")
+    issue_type: IssueTypeLiteral = Field(description="Machine-readable issue category")
+    severity: Severity = Field(description="Three-tier severity")
+    confidence: float = Field(ge=0.0, le=1.0, description="Detector confidence")
+    expected: str | None = Field(default=None, description="Expected value (if known)")
+    actual: str = Field(description="Actual value found in the cell")
+    reason: str = Field(min_length=1, description="Human-readable explanation")
+
+    model_config = {"frozen": True}
+
+
+class Detector(Protocol):
+    """Structural protocol that every detector must implement.
+
+    A detector is a pure function over tabular data: it receives a table
+    and an optional Schema, and returns a list of Issue objects. No LLM calls,
+    no disk I/O, no side effects.
+
+    Example:
+        >>> class MyDetector:
+        ...     def detect(
+        ...         self, df: TableLike, schema: Schema | None = None
+        ...     ) -> list[Issue]:
+        ...         return []
+    """
+
+    def detect(self, df: TableLike, schema: Schema | None = None) -> list[Issue]:
+        """Detect data-quality issues in the given DataFrame.
+
+        Args:
+            df: The input table to analyze.
+            schema: Optional declared schema with column types and constraints.
+
+        Returns:
+            A list of Issue objects describing detected anomalies.
+        """
+        ...  # pragma: no cover

dataforge/detectors/decimal_shift.py

@@ -0,0 +1,166 @@
+"""Detector for decimal-shift anomalies in numeric columns.
+
+Identifies values that are exact powers-of-10 multiples (10x, 100x, 0.1x,
+0.01x, etc.) of the column's central tendency.  This is the canonical
+"decimal point was moved" data-entry error pattern.
+
+The detector is **pure**: no LLM calls, no I/O, no side effects.
+"""
+
+from __future__ import annotations
+
+import math
+from statistics import median
+
+from dataforge.detectors.base import Issue, Schema, Severity
+from dataforge.table import TableLike, column_names, column_values
+
+# Minimum non-null numeric values required for meaningful statistics.
+_MIN_COLUMN_SIZE = 5
+
+# Powers of 10 to check.  Positive = value is N× too large;
+# negative = value is N× too small.
+_SHIFT_POWERS = (-3, -2, -1, 1, 2, 3)
+
+# How close ratio must be to a power of 10 (in log10 space).
+# 0.15 means we accept ratios within 10^±0.15 ≈ 0.71× – 1.41× of the
+# exact power.  Tight enough to avoid false positives on natural variance.
+_LOG_TOLERANCE = 0.15
+
+
+def _try_float(value: object) -> float | None:
+    """Attempt to parse a value as float, returning None on failure.
+
+    Args:
+        value: Any value (string, int, float, None, …).
+
+    Returns:
+        The float value or None if parsing fails.
+    """
+    if value is None or (isinstance(value, float) and math.isnan(value)):
+        return None
+    try:
+        return float(str(value))
+    except (ValueError, TypeError):
+        return None
+
+
+class DecimalShiftDetector:
+    """Detects values that are power-of-10 multiples of the column distribution.
+
+    For each numeric column, computes the median and checks every value
+    to see if ``value / median`` is close to 10^k for k in {-3, -2, -1,
+    1, 2, 3}.  Flagged values get an ``expected`` field with the corrected
+    value (``value / 10^k``).
+
+    Requires at least 5 non-null numeric values per column.  Columns with
+    zero or near-zero median are handled gracefully.
+
+    Example:
+        >>> import pandas as pd
+        >>> detector = DecimalShiftDetector()
+        >>> df = pd.DataFrame({"price": [100.0, 105.0, 98.0, 1020.0, 103.0]})
+        >>> issues = detector.detect(df)
+        >>> issues[0].row
+        3
+    """
+
+    def detect(self, df: TableLike, schema: Schema | None = None) -> list[Issue]:
+        """Detect decimal-shift issues in the DataFrame.
+
+        Args:
+            df: The input DataFrame to analyze.
+            schema: Optional declared schema (unused by this detector).
+
+        Returns:
+            A list of Issue objects for values that appear to be shifted
+            by a power of 10 relative to the column distribution.
+        """
+        issues: list[Issue] = []
+
+        for col_name in column_names(df):
+            col_issues = self._check_column(df, str(col_name))
+            issues.extend(col_issues)
+
+        return issues
+
+    def _check_column(self, df: TableLike, col_name: str) -> list[Issue]:
+        """Check a single column for decimal-shift outliers.
+
+        Args:
+            df: The DataFrame containing the column.
+            col_name: Name of the column to check.
+
+        Returns:
+            Issues found in this column.
+        """
+        # Parse all values to float, keeping track of original indices.
+        parsed: list[tuple[int, float, str]] = []
+        for row_idx, val in enumerate(column_values(df, col_name)):
+            fval = _try_float(val)
+            if fval is not None:
+                parsed.append((row_idx, fval, str(val)))
+
+        if len(parsed) < _MIN_COLUMN_SIZE:
+            return []
+
+        center = float(median([v for _, v, _ in parsed]))
+
+        # If median is zero or very close, we cannot compute meaningful ratios.
+        if abs(center) < 1e-10:
+            return []
+
+        issues: list[Issue] = []
+        for row_idx, fval, str_val in parsed:
+            if abs(fval) < 1e-10:
+                continue
+
+            ratio = fval / center
+            if abs(ratio) < 1e-10:
+                continue
+
+            log_ratio = math.log10(abs(ratio))
+
+            best_power: int | None = None
+            best_distance = float("inf")
+
+            for power in _SHIFT_POWERS:
+                distance = abs(log_ratio - power)
+                if distance < _LOG_TOLERANCE and distance < best_distance:
+                    best_distance = distance
+                    best_power = power
+
+            if best_power is not None:
+                correction_factor = 10.0**best_power
+                expected_val = fval / correction_factor
+
+                # Confidence: closer to exact power → higher confidence.
+                confidence = round(min(0.95, max(0.70, 1.0 - best_distance * 2.0)), 2)
+
+                if best_power > 0:
+                    reason = (
+                        f"Value {fval:g} in column '{col_name}' appears to be "
+                        f"~{int(correction_factor)}x the typical value "
+                        f"(median ~{center:g})"
+                    )
+                else:
+                    reason = (
+                        f"Value {fval:g} in column '{col_name}' appears to be "
+                        f"~{1.0 / correction_factor:g}x too small compared to "
+                        f"the typical value (median ~{center:g})"
+                    )
+
+                issues.append(
+                    Issue(
+                        row=row_idx,
+                        column=col_name,
+                        issue_type="decimal_shift",
+                        severity=Severity.REVIEW,
+                        confidence=confidence,
+                        expected=f"{expected_val:g}",
+                        actual=str_val.strip(),
+                        reason=reason,
+                    )
+                )
+
+        return issues

dataforge/detectors/fd_violation.py

@@ -0,0 +1,157 @@
+"""Detector for functional-dependency violations in tabular data.
+
+Given a declared functional dependency X -> Y (where X is a set of
+determinant columns and Y is a dependent column), this detector groups
+rows by X and flags any group where Y takes more than one distinct value.
+
+Week 1 scope: declared FDs only (from the schema YAML).  Automatic FD
+mining is deferred to a later milestone.
+
+The detector is **pure**: no LLM calls, no I/O, no side effects.
+"""
+
+from __future__ import annotations
+
+from dataforge.detectors.base import Issue, Schema, Severity
+from dataforge.table import TableLike, cell_value, column_names, row_count
+
+
+class FDViolationDetector:
+    """Detects rows that violate declared functional dependencies.
+
+    For each FD ``determinant -> dependent`` in the schema, groups the
+    DataFrame by the determinant columns and checks that each group has
+    exactly one unique value in the dependent column.  All rows in a
+    violating group are flagged.
+
+    Requires a ``Schema`` with ``functional_dependencies`` to do anything;
+    returns an empty list if no schema or no FDs are provided.
+
+    Example:
+        >>> import pandas as pd
+        >>> from dataforge.detectors.base import FunctionalDependency, Schema
+        >>> detector = FDViolationDetector()
+        >>> df = pd.DataFrame({
+        ...     "zip": ["10001", "10001", "90210"],
+        ...     "city": ["NY", "Manhattan", "LA"],
+        ... })
+        >>> schema = Schema(functional_dependencies=[
+        ...     FunctionalDependency(determinant=["zip"], dependent="city"),
+        ... ])
+        >>> issues = detector.detect(df, schema)
+        >>> len(issues)
+        2
+    """
+
+    def detect(self, df: TableLike, schema: Schema | None = None) -> list[Issue]:
+        """Detect FD-violation issues in the DataFrame.
+
+        Args:
+            df: The input DataFrame to analyze.
+            schema: Schema containing declared functional dependencies.
+                If None or no FDs declared, returns an empty list.
+
+        Returns:
+            A list of Issue objects for rows violating declared FDs.
+        """
+        if schema is None or not schema.functional_dependencies:
+            return []
+
+        issues: list[Issue] = []
+
+        for fd in schema.functional_dependencies:
+            fd_issues = self._check_fd(df, fd.determinant, fd.dependent)
+            issues.extend(fd_issues)
+
+        return issues
+
+    def _check_fd(
+        self,
+        df: TableLike,
+        determinant: tuple[str, ...],
+        dependent: str,
+    ) -> list[Issue]:
+        """Check a single functional dependency X -> Y.
+
+        Args:
+            df: The DataFrame to check.
+            determinant: List of determinant column names (X).
+            dependent: The dependent column name (Y).
+
+        Returns:
+            Issues for all rows in groups that violate the FD.
+        """
+        determinant_columns = list(determinant)
+
+        # Verify all columns exist in the DataFrame.
+        all_cols = [*determinant_columns, dependent]
+        available_columns = set(column_names(df))
+        for col in all_cols:
+            if col not in available_columns:
+                return []
+
+        groups: dict[tuple[str, ...], list[int]] = {}
+        for row in range(row_count(df)):
+            group_key = tuple(cell_value(df, row, column) for column in determinant_columns)
+            if any(value == "" for value in group_key):
+                continue
+            groups.setdefault(group_key, []).append(row)
+
+        if not groups:
+            return []
+
+        issues: list[Issue] = []
+        for group_key, row_indices in groups.items():
+            unique_deps: list[str] = []
+            for row in row_indices:
+                value = cell_value(df, row, dependent)
+                if value == "" or value in unique_deps:
+                    continue
+                unique_deps.append(value)
+            if len(unique_deps) <= 1:
+                continue
+
+            det_desc = self._format_determinant(determinant, group_key)
+            unique_str = ", ".join(repr(str(v)) for v in unique_deps)
+
+            for idx in row_indices:
+                actual_val = cell_value(df, idx, dependent)
+                reason = (
+                    f"Functional dependency {determinant} -> {dependent} "
+                    f"violated: {det_desc} maps to multiple values: "
+                    f"{{{unique_str}}}"
+                )
+                issues.append(
+                    Issue(
+                        row=int(idx),
+                        column=dependent,
+                        issue_type="fd_violation",
+                        severity=Severity.UNSAFE,
+                        confidence=0.95,
+                        actual=actual_val,
+                        reason=reason,
+                    )
+                )
+
+        return issues
+
+    @staticmethod
+    def _format_determinant(determinant: tuple[str, ...], group_key: object) -> str:
+        """Format the determinant key for human-readable output.
+
+        Args:
+            determinant: List of determinant column names.
+            group_key: The group key (scalar or tuple).
+
+        Returns:
+            A formatted string like ``zip_code='10001'``.
+        """
+        if len(determinant) == 1:
+            return f"{determinant[0]}='{group_key}'"
+
+        # Composite key: group_key is a tuple.
+        if isinstance(group_key, tuple):
+            parts = [f"{col}='{val}'" for col, val in zip(determinant, group_key, strict=True)]
+            return ", ".join(parts)
+
+        return f"{determinant}='{group_key}'"