dataforge-07 0.1.0__py3-none-any.whl

dataforge/causal/dag.py

@@ -0,0 +1,174 @@
+"""Column-level causal DAG utilities for root-cause analysis."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import networkx as nx  # type: ignore[import-untyped]
+
+__all__ = ["CausalDAG", "CausalEdge"]
+
+
+@dataclass(frozen=True)
+class CausalEdge:
+    """Metadata for a directed causal edge.
+
+    Args:
+        source: Source column name.
+        target: Target column name.
+        confidence: Confidence in the directed influence, from 0.0 to 1.0.
+        provenance: Human-readable source of the edge.
+    """
+
+    source: str
+    target: str
+    confidence: float
+    provenance: str
+
+
+class CausalDAG:
+    """Acyclic directed graph whose nodes are dataset columns.
+
+    Args:
+        nodes: Optional initial column names.
+
+    Example:
+        >>> dag = CausalDAG(["discount_pct", "order_total"])
+        >>> dag.add_edge("discount_pct", "order_total", confidence=0.9, provenance="fd")
+        >>> dag.is_reachable("discount_pct", "order_total")
+        True
+    """
+
+    def __init__(self, nodes: list[str] | tuple[str, ...] = ()) -> None:
+        self._graph: nx.DiGraph[Any] = nx.DiGraph()
+        self._graph.add_nodes_from(nodes)
+
+    @property
+    def nodes(self) -> tuple[str, ...]:
+        """Return graph nodes in insertion order."""
+        return tuple(str(node) for node in self._graph.nodes)
+
+    @property
+    def edges(self) -> tuple[CausalEdge, ...]:
+        """Return directed edges with metadata."""
+        result: list[CausalEdge] = []
+        for source, target, attrs in self._graph.edges(data=True):
+            result.append(
+                CausalEdge(
+                    source=str(source),
+                    target=str(target),
+                    confidence=float(attrs.get("confidence", 0.0)),
+                    provenance=str(attrs.get("provenance", "unknown")),
+                )
+            )
+        return tuple(result)
+
+    def add_node(self, column: str) -> None:
+        """Add a column node if it is not already present.
+
+        Args:
+            column: Column name.
+        """
+        self._graph.add_node(column)
+
+    def add_edge(
+        self,
+        source: str,
+        target: str,
+        *,
+        confidence: float,
+        provenance: str,
+    ) -> None:
+        """Add a directed causal edge while preserving acyclicity.
+
+        Args:
+            source: Source column name.
+            target: Target column name.
+            confidence: Confidence score from 0.0 to 1.0.
+            provenance: Source of the edge.
+
+        Raises:
+            ValueError: If the edge is self-referential or creates a cycle.
+        """
+        if source == target:
+            raise ValueError("Causal DAG does not allow self-edges")
+        self._graph.add_node(source)
+        self._graph.add_node(target)
+        if nx.has_path(self._graph, target, source):
+            raise ValueError(f"Adding {source!r} -> {target!r} would create a cycle")
+        bounded = max(0.0, min(1.0, confidence))
+        self._graph.add_edge(source, target, confidence=bounded, provenance=provenance)
+
+    def successors(self, column: str) -> tuple[str, ...]:
+        """Return direct downstream columns for a node.
+
+        Args:
+            column: Column name.
+
+        Returns:
+            A tuple of direct successor column names.
+        """
+        if column not in self._graph:
+            return ()
+        return tuple(str(node) for node in self._graph.successors(column))
+
+    def is_reachable(self, source: str, target: str) -> bool:
+        """Return whether target is reachable from source.
+
+        Args:
+            source: Source column name.
+            target: Target column name.
+
+        Returns:
+            True if source equals target or a directed path exists.
+        """
+        if source == target:
+            return True
+        if source not in self._graph or target not in self._graph:
+            return False
+        return bool(nx.has_path(self._graph, source, target))
+
+    def path_confidence(self, source: str, target: str) -> float:
+        """Return the weakest-edge confidence on the shortest path.
+
+        Args:
+            source: Source column name.
+            target: Target column name.
+
+        Returns:
+            Confidence in [0.0, 1.0], or 0.0 when no path exists.
+        """
+        if source == target:
+            return 1.0
+        if not self.is_reachable(source, target):
+            return 0.0
+        path = nx.shortest_path(self._graph, source, target)
+        confidences = [
+            float(self._graph.edges[path[i], path[i + 1]].get("confidence", 0.0))
+            for i in range(len(path) - 1)
+        ]
+        return min(confidences, default=0.0)
+
+    def minimal_root_columns(self, columns: list[str] | tuple[str, ...]) -> tuple[str, ...]:
+        """Return selected columns that are not downstream of another selection.
+
+        Args:
+            columns: Selected error columns.
+
+        Returns:
+            Minimal root columns in first-seen order.
+        """
+        unique: list[str] = []
+        for column in columns:
+            if column not in unique:
+                unique.append(column)
+
+        roots: list[str] = []
+        for column in unique:
+            has_upstream = any(
+                other != column and self.is_reachable(other, column) for other in unique
+            )
+            if not has_upstream:
+                roots.append(column)
+        return tuple(roots)

dataforge/causal/pc.py

@@ -0,0 +1,232 @@
+"""PC-based causal DAG discovery with functional-dependency priors."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from scipy.stats import chi2_contingency  # type: ignore[import-untyped]
+
+from dataforge.causal.dag import CausalDAG
+from dataforge.verifier.schema import Schema
+
+__all__ = ["CausalDiscoveryResult", "discover_causal_dag"]
+
+
+@dataclass(frozen=True)
+class CausalDiscoveryResult:
+    """Result of causal discovery.
+
+    Args:
+        dag: Directed acyclic graph over columns.
+        confidence_report: Column-pair confidence or diagnostic metadata.
+        warnings: Non-fatal discovery warnings.
+    """
+
+    dag: CausalDAG
+    confidence_report: dict[str, float] = field(default_factory=dict)
+    warnings: tuple[str, ...] = ()
+
+
+def discover_causal_dag(
+    df: pd.DataFrame,
+    schema: Schema | None = None,
+    *,
+    alpha: float = 0.05,
+) -> CausalDiscoveryResult:
+    """Infer a deterministic causal DAG from tabular data and FD priors.
+
+    Args:
+        df: Input DataFrame.
+        schema: Optional declared schema with functional dependencies.
+        alpha: Significance threshold for independence checks.
+
+    Returns:
+        CausalDiscoveryResult. A DAG is returned even if PC orientation is
+        underdetermined; low-confidence edges are tagged as such.
+    """
+    columns = [str(column) for column in df.columns]
+    dag = CausalDAG(columns)
+    report: dict[str, float] = {}
+    warnings: list[str] = []
+
+    if schema is not None:
+        for fd in schema.functional_dependencies:
+            for determinant in fd.determinant:
+                _try_add_edge(
+                    dag,
+                    determinant,
+                    fd.dependent,
+                    confidence=0.95,
+                    provenance="functional_dependency_prior",
+                    warnings=warnings,
+                )
+                report[f"{determinant}->{fd.dependent}"] = 0.95
+
+    cleaned = _prepare_for_pc(df)
+    pc_edges, pc_warning = _run_causal_learn_pc(cleaned.to_numpy(), columns, alpha)
+    if pc_warning:
+        warnings.append(pc_warning)
+    for source, target in pc_edges:
+        _try_add_edge(
+            dag,
+            source,
+            target,
+            confidence=0.55,
+            provenance="causal_learn_pc",
+            warnings=warnings,
+        )
+        report.setdefault(f"{source}->{target}", 0.55)
+
+    for source, target, confidence in _pairwise_dependency_edges(df, alpha):
+        _try_add_edge(
+            dag,
+            source,
+            target,
+            confidence=confidence,
+            provenance="pairwise_ci_fallback",
+            warnings=warnings,
+        )
+        report.setdefault(f"{source}->{target}", confidence)
+
+    return CausalDiscoveryResult(dag=dag, confidence_report=report, warnings=tuple(warnings))
+
+
+def _prepare_for_pc(df: pd.DataFrame) -> pd.DataFrame:
+    """Return numeric data with no NaN values for causal-learn PC."""
+    prepared = pd.DataFrame(index=df.index)
+    for column in df.columns:
+        numeric = pd.to_numeric(df[column], errors="coerce")
+        if numeric.notna().sum() >= max(2, int(0.5 * len(df))):
+            fill = float(numeric.median()) if numeric.notna().any() else 0.0
+            prepared[str(column)] = numeric.fillna(fill)
+        else:
+            codes, _ = pd.factorize(df[column].astype("string").fillna("<missing>"), sort=True)
+            prepared[str(column)] = codes.astype(float)
+    return prepared.fillna(0.0)
+
+
+def _run_causal_learn_pc(
+    data: np.ndarray[Any, Any], columns: list[str], alpha: float
+) -> tuple[list[tuple[str, str]], str | None]:
+    """Run causal-learn PC and return deterministic directed edges."""
+    try:
+        from causallearn.search.ConstraintBased.PC import pc  # type: ignore[import-untyped]
+
+        result = pc(data, alpha=alpha, indep_test="fisherz", stable=True, show_progress=False)
+    except Exception as exc:
+        return [], f"causal-learn PC unavailable or failed: {exc}"
+
+    matrix = getattr(getattr(result, "G", None), "graph", None)
+    if matrix is None:
+        return [], "causal-learn PC returned no adjacency matrix"
+
+    edges: list[tuple[str, str]] = []
+    arr = np.asarray(matrix)
+    for i, source in enumerate(columns):
+        for j, target in enumerate(columns):
+            if i >= j or i >= arr.shape[0] or j >= arr.shape[1]:
+                continue
+            if arr[i, j] != 0 or arr[j, i] != 0:
+                edges.append((source, target))
+    return edges, None
+
+
+def _pairwise_dependency_edges(df: pd.DataFrame, alpha: float) -> list[tuple[str, str, float]]:
+    """Return deterministic low-confidence edges for dependent column pairs."""
+    columns = [str(column) for column in df.columns]
+    edges: list[tuple[str, str, float]] = []
+    for i, source in enumerate(columns):
+        for target in columns[i + 1 :]:
+            p_value = _pairwise_p_value(df[source], df[target])
+            if p_value < alpha:
+                confidence = max(0.25, min(0.75, 1.0 - p_value))
+                edges.append((source, target, round(confidence, 4)))
+    return edges
+
+
+def _pairwise_p_value(left: pd.Series[Any], right: pd.Series[Any]) -> float:
+    """Return a p-value using categorical, continuous, or mixed tests."""
+    left_numeric = pd.to_numeric(left, errors="coerce")
+    right_numeric = pd.to_numeric(right, errors="coerce")
+    left_cont = left_numeric.notna().sum() >= max(5, int(0.8 * len(left)))
+    right_cont = right_numeric.notna().sum() >= max(5, int(0.8 * len(right)))
+
+    if left_cont and right_cont:
+        return _hsic_p_value(
+            left_numeric.fillna(left_numeric.median()), right_numeric.fillna(right_numeric.median())
+        )
+    if not left_cont and not right_cont:
+        return _chi_squared_p_value(left, right)
+    return _mutual_information_p_value(left, right)
+
+
+def _chi_squared_p_value(left: pd.Series[Any], right: pd.Series[Any]) -> float:
+    """Return chi-squared independence p-value for categorical pairs."""
+    table = pd.crosstab(
+        left.astype("string").fillna("<missing>"), right.astype("string").fillna("<missing>")
+    )
+    if table.shape[0] < 2 or table.shape[1] < 2:
+        return 1.0
+    _, p_value, _, _ = chi2_contingency(table)
+    return float(p_value)
+
+
+def _hsic_p_value(left: pd.Series[Any], right: pd.Series[Any]) -> float:
+    """Return HSIC p-value for continuous pairs, with correlation fallback."""
+    x = left.to_numpy(dtype=float).reshape(-1, 1)
+    y = right.to_numpy(dtype=float).reshape(-1, 1)
+    try:
+        from hyppo.independence import Hsic  # type: ignore[import-untyped]
+
+        _, p_value = Hsic().test(x, y, reps=100, auto=True)
+        return float(p_value)
+    except Exception:
+        corr = abs(float(np.corrcoef(x[:, 0], y[:, 0])[0, 1]))
+        return 0.0 if corr > 0.75 else 1.0
+
+
+def _mutual_information_p_value(left: pd.Series[Any], right: pd.Series[Any]) -> float:
+    """Return a bounded pseudo p-value from binned mutual information."""
+    left_codes = _codes(left)
+    right_codes = _codes(right)
+    table = pd.crosstab(left_codes, right_codes)
+    total = float(table.to_numpy().sum())
+    if total == 0.0 or table.shape[0] < 2 or table.shape[1] < 2:
+        return 1.0
+    joint = table.to_numpy(dtype=float) / total
+    px = joint.sum(axis=1, keepdims=True)
+    py = joint.sum(axis=0, keepdims=True)
+    expected = px @ py
+    mask = joint > 0
+    mi = float((joint[mask] * np.log(joint[mask] / expected[mask])).sum())
+    return float(np.exp(-mi))
+
+
+def _codes(series: pd.Series[Any]) -> np.ndarray[Any, Any]:
+    """Return stable integer codes for a mixed-type series."""
+    numeric = pd.to_numeric(series, errors="coerce")
+    if numeric.notna().sum() >= max(5, int(0.8 * len(series))):
+        return pd.qcut(
+            numeric.fillna(numeric.median()), q=4, duplicates="drop"
+        ).cat.codes.to_numpy()
+    codes, _ = pd.factorize(series.astype("string").fillna("<missing>"), sort=True)
+    return codes
+
+
+def _try_add_edge(
+    dag: CausalDAG,
+    source: str,
+    target: str,
+    *,
+    confidence: float,
+    provenance: str,
+    warnings: list[str],
+) -> None:
+    """Add an edge or record the cycle warning."""
+    try:
+        dag.add_edge(source, target, confidence=confidence, provenance=provenance)
+    except ValueError as exc:
+        warnings.append(str(exc))

dataforge/causal/root_cause.py

@@ -0,0 +1,193 @@
+"""Minimal root-cause selection over detected errors and a causal DAG."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+from pydantic import BaseModel, Field
+
+from dataforge.causal.dag import CausalDAG
+
+__all__ = [
+    "CausalRootCauseAnalyzer",
+    "ErrorEvidence",
+    "RootCauseResult",
+    "evidence_from_issue",
+    "minimal_root_set",
+]
+
+
+class _IssueLike(Protocol):
+    """Protocol for objects with row/column issue fields."""
+
+    row: int
+    column: str
+    issue_type: str
+
+
+class ErrorEvidence(BaseModel):
+    """Column-mapped detected error used for causal root-cause analysis.
+
+    Args:
+        index: Zero-based error index in the caller's selected issue list.
+        row: Row index where the error was detected.
+        column: Column where the error was detected.
+        issue_type: Machine-readable issue type.
+    """
+
+    index: int = Field(ge=0)
+    row: int = Field(ge=0)
+    column: str = Field(min_length=1)
+    issue_type: str = Field(min_length=1)
+
+    model_config = {"frozen": True}
+
+
+class RootCauseResult(BaseModel):
+    """Structured result returned by the root-cause analyzer.
+
+    Args:
+        root_indices: Minimal selected error indices.
+        root_columns: Root columns corresponding to root_indices.
+        covered_indices: Selected error indices covered by the root set.
+        confidence: Mean path confidence from roots to covered errors.
+        explanation: Human-readable explanation of the selected roots.
+    """
+
+    root_indices: list[int]
+    root_columns: list[str]
+    covered_indices: list[int]
+    confidence: float
+    explanation: str
+
+    model_config = {"frozen": True}
+
+
+class CausalRootCauseAnalyzer:
+    """Compute minimal root causes for selected detected errors.
+
+    Args:
+        dag: Column-level causal DAG.
+
+    Example:
+        >>> dag = CausalDAG(["discount_pct", "order_total"])
+        >>> dag.add_edge("discount_pct", "order_total", confidence=0.9, provenance="formula")
+        >>> errors = [
+        ...     ErrorEvidence(index=0, row=1, column="discount_pct", issue_type="bad"),
+        ...     ErrorEvidence(index=1, row=1, column="order_total", issue_type="bad"),
+        ... ]
+        >>> CausalRootCauseAnalyzer(dag).analyze(errors).root_indices
+        [0]
+    """
+
+    def __init__(self, dag: CausalDAG) -> None:
+        self._dag = dag
+
+    def analyze(self, errors: list[ErrorEvidence] | tuple[ErrorEvidence, ...]) -> RootCauseResult:
+        """Return the minimal root set for the selected errors.
+
+        Args:
+            errors: Selected detected errors.
+
+        Returns:
+            RootCauseResult with roots, coverage, confidence, and explanation.
+        """
+        if not errors:
+            return RootCauseResult(
+                root_indices=[],
+                root_columns=[],
+                covered_indices=[],
+                confidence=0.0,
+                explanation="No errors were supplied.",
+            )
+
+        roots: list[ErrorEvidence] = []
+        for candidate in errors:
+            if not self._has_upstream_selected_error(candidate, errors):
+                roots.append(candidate)
+
+        covered: list[int] = []
+        path_confidences: list[float] = []
+        for error in errors:
+            for root in roots:
+                if root.column == error.column or self._dag.is_reachable(root.column, error.column):
+                    covered.append(error.index)
+                    path_confidences.append(self._dag.path_confidence(root.column, error.column))
+                    break
+
+        confidence = (
+            round(sum(path_confidences) / len(path_confidences), 4) if path_confidences else 0.0
+        )
+        root_columns = [root.column for root in roots]
+        return RootCauseResult(
+            root_indices=[root.index for root in roots],
+            root_columns=root_columns,
+            covered_indices=covered,
+            confidence=confidence,
+            explanation=self._explain(root_columns, len(covered), len(errors)),
+        )
+
+    def _has_upstream_selected_error(
+        self,
+        candidate: ErrorEvidence,
+        errors: list[ErrorEvidence] | tuple[ErrorEvidence, ...],
+    ) -> bool:
+        """Return whether another selected error causally precedes candidate."""
+        for other in errors:
+            if other.index == candidate.index:
+                continue
+            if other.column == candidate.column and other.index < candidate.index:
+                return True
+            if other.column != candidate.column and self._dag.is_reachable(
+                other.column, candidate.column
+            ):
+                return True
+        return False
+
+    @staticmethod
+    def _explain(root_columns: list[str], covered_count: int, total_count: int) -> str:
+        """Build a compact result explanation."""
+        if not root_columns:
+            return "No minimal roots were found."
+        joined = ", ".join(root_columns)
+        return f"Selected {joined} as minimal roots covering {covered_count}/{total_count} errors."
+
+
+def minimal_root_set(
+    errors: list[ErrorEvidence] | tuple[ErrorEvidence, ...], dag: CausalDAG
+) -> RootCauseResult:
+    """Convenience wrapper for CausalRootCauseAnalyzer.
+
+    Args:
+        errors: Selected detected errors.
+        dag: Column-level causal DAG.
+
+    Returns:
+        Minimal root-cause result.
+    """
+    return CausalRootCauseAnalyzer(dag).analyze(errors)
+
+
+def evidence_from_issue(index: int, issue: _IssueLike | dict[str, Any]) -> ErrorEvidence:
+    """Build ErrorEvidence from an Issue-like object or dictionary.
+
+    Args:
+        index: Error index to assign.
+        issue: Object or dictionary with row/column/type fields.
+
+    Returns:
+        ErrorEvidence instance.
+    """
+    if isinstance(issue, dict):
+        return ErrorEvidence(
+            index=index,
+            row=int(issue.get("row", 0)),
+            column=str(issue.get("column", "")),
+            issue_type=str(issue.get("type", issue.get("issue_type", "unknown"))),
+        )
+    return ErrorEvidence(
+        index=index,
+        row=int(issue.row),
+        column=str(issue.column),
+        issue_type=str(issue.issue_type),
+    )

dataforge/cli/__init__.py

@@ -0,0 +1,50 @@
+"""Typer application entrypoint for DataForge.
+
+Each CLI subcommand is defined in its own module under ``dataforge.cli.*``
+and registered here. The ``app`` object is the entry point referenced by
+``[project.scripts]`` in ``pyproject.toml``.
+"""
+
+import typer
+
+from dataforge.cli.audit import audit
+from dataforge.cli.bench import bench
+from dataforge.cli.constraints import constraints_app
+from dataforge.cli.profile import profile
+from dataforge.cli.release import release_app
+from dataforge.cli.repair import repair
+from dataforge.cli.revert import revert
+from dataforge.cli.watch import watch
+
+app: typer.Typer = typer.Typer(
+    help="DataForge - AI-powered data-quality detection and repair.",
+    no_args_is_help=True,
+)
+
+
+@app.callback(invoke_without_command=True)
+def _main(
+    version: bool = typer.Option(
+        False,
+        "--version",
+        "-V",
+        help="Show version and exit.",
+        is_eager=True,
+    ),
+) -> None:
+    """DataForge - AI-powered data-quality detection and repair."""
+    if version:
+        from dataforge import __version__
+
+        typer.echo(f"dataforge {__version__}")
+        raise typer.Exit()
+
+
+app.command(name="profile")(profile)
+app.command(name="repair")(repair)
+app.command(name="revert")(revert)
+app.command(name="audit")(audit)
+app.command(name="bench")(bench)
+app.command(name="watch")(watch)
+app.add_typer(constraints_app, name="constraints")
+app.add_typer(release_app, name="release")