duckguard 2.0.0__py3-none-any.whl → 2.2.0__py3-none-any.whl

duckguard/connectors/postgres.py

@@ -2,10 +2,9 @@
 
 from __future__ import annotations
 
-import re
 from urllib.parse import urlparse
 
-from duckguard.connectors.base import Connector, ConnectionConfig
+from duckguard.connectors.base import ConnectionConfig, Connector
 from duckguard.core.dataset import Dataset
 from duckguard.core.engine import DuckGuardEngine
 

duckguard/connectors/redshift.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from urllib.parse import urlparse
 
-from duckguard.connectors.base import Connector, ConnectionConfig
+from duckguard.connectors.base import ConnectionConfig, Connector
 from duckguard.core.dataset import Dataset
 from duckguard.core.engine import DuckGuardEngine
 

duckguard/connectors/snowflake.py

@@ -2,11 +2,10 @@
 
 from __future__ import annotations
 
-import re
 from typing import Any
 from urllib.parse import parse_qs, urlparse
 
-from duckguard.connectors.base import Connector, ConnectionConfig
+from duckguard.connectors.base import ConnectionConfig, Connector
 from duckguard.core.dataset import Dataset
 from duckguard.core.engine import DuckGuardEngine
 

duckguard/connectors/sqlite.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import os
 from pathlib import Path
 
-from duckguard.connectors.base import Connector, ConnectionConfig
+from duckguard.connectors.base import ConnectionConfig, Connector
 from duckguard.core.dataset import Dataset
 from duckguard.core.engine import DuckGuardEngine
 

duckguard/connectors/sqlserver.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from typing import Any
 from urllib.parse import parse_qs, urlparse
 
-from duckguard.connectors.base import Connector, ConnectionConfig
+from duckguard.connectors.base import ConnectionConfig, Connector
 from duckguard.core.dataset import Dataset
 from duckguard.core.engine import DuckGuardEngine
 
@@ -55,20 +55,17 @@ class SQLServerConnector(Connector):
             Dataset object
         """
         # Try pyodbc first, then pymssql
-        try:
-            import pyodbc
+        import importlib.util
 
+        if importlib.util.find_spec("pyodbc") is not None:
             driver_module = "pyodbc"
-        except ImportError:
-            try:
-                import pymssql
-
-                driver_module = "pymssql"
-            except ImportError:
-                raise ImportError(
-                    "SQL Server support requires pyodbc or pymssql. "
-                    "Install with: pip install duckguard[sqlserver]"
-                )
+        elif importlib.util.find_spec("pymssql") is not None:
+            driver_module = "pymssql"
+        else:
+            raise ImportError(
+                "SQL Server support requires pyodbc or pymssql. "
+                "Install with: pip install duckguard[sqlserver]"
+            )
 
         if not config.table:
             raise ValueError("Table name is required for SQL Server connections")

duckguard/contracts/__init__.py

@@ -14,17 +14,17 @@ Example:
         print(f"Contract violations: {result.violations}")
 """
 
+from duckguard.contracts.diff import SchemaDiff, diff_contracts
+from duckguard.contracts.generator import generate_contract
+from duckguard.contracts.loader import contract_to_yaml, load_contract, load_contract_from_string
 from duckguard.contracts.schema import (
+    ContractMetadata,
     DataContract,
-    SchemaField,
     FieldType,
     QualitySLA,
-    ContractMetadata,
+    SchemaField,
 )
-from duckguard.contracts.loader import load_contract, load_contract_from_string, contract_to_yaml
-from duckguard.contracts.validator import validate_contract, ContractValidationResult
-from duckguard.contracts.generator import generate_contract
-from duckguard.contracts.diff import diff_contracts, SchemaDiff
+from duckguard.contracts.validator import ContractValidationResult, validate_contract
 
 __all__ = [
     # Schema

duckguard/contracts/diff.py

@@ -9,7 +9,7 @@ from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any
 
-from duckguard.contracts.schema import DataContract, SchemaField, FieldType
+from duckguard.contracts.schema import DataContract, FieldType, SchemaField
 
 
 class ChangeType(Enum):

duckguard/contracts/generator.py

@@ -7,19 +7,18 @@ from __future__ import annotations
 
 from datetime import datetime
 from pathlib import Path
-from typing import Any
 
-from duckguard.core.dataset import Dataset
 from duckguard.connectors import connect
+from duckguard.contracts.loader import contract_to_yaml
 from duckguard.contracts.schema import (
+    ContractMetadata,
     DataContract,
-    SchemaField,
-    FieldType,
     FieldConstraint,
+    FieldType,
     QualitySLA,
-    ContractMetadata,
+    SchemaField,
 )
-from duckguard.contracts.loader import contract_to_yaml
+from duckguard.core.dataset import Dataset
 from duckguard.semantic import SemanticAnalyzer, SemanticType
 
 

duckguard/contracts/loader.py

@@ -47,12 +47,12 @@ from typing import Any
 import yaml
 
 from duckguard.contracts.schema import (
+    ContractMetadata,
     DataContract,
-    SchemaField,
-    FieldType,
     FieldConstraint,
+    FieldType,
     QualitySLA,
-    ContractMetadata,
+    SchemaField,
 )
 
 
@@ -82,7 +82,7 @@ def load_contract(path: str | Path) -> DataContract:
     if not path.exists():
         raise FileNotFoundError(f"Contract file not found: {path}")
 
-    with open(path, "r", encoding="utf-8") as f:
+    with open(path, encoding="utf-8") as f:
         content = f.read()
 
     return load_contract_from_string(content, source_file=str(path))

duckguard/contracts/validator.py

@@ -6,14 +6,13 @@ Validates datasets against data contracts to ensure compliance.
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from datetime import datetime, timedelta
+from datetime import datetime
 from enum import Enum
 from typing import Any
-import re
 
-from duckguard.core.dataset import Dataset
 from duckguard.connectors import connect
-from duckguard.contracts.schema import DataContract, SchemaField, FieldType
+from duckguard.contracts.schema import DataContract, SchemaField
+from duckguard.core.dataset import Dataset
 
 
 class ViolationType(Enum):

duckguard/core/__init__.py

@@ -1,8 +1,8 @@
 """Core module containing the engine, dataset, and column classes."""
 
-from duckguard.core.engine import DuckGuardEngine
-from duckguard.core.dataset import Dataset
 from duckguard.core.column import Column
-from duckguard.core.result import ValidationResult, CheckResult
+from duckguard.core.dataset import Dataset
+from duckguard.core.engine import DuckGuardEngine
+from duckguard.core.result import CheckResult, ValidationResult
 
 __all__ = ["DuckGuardEngine", "Dataset", "Column", "ValidationResult", "CheckResult"]

duckguard/core/column.py

@@ -2,14 +2,16 @@
 
 from __future__ import annotations
 
-import re
 from typing import TYPE_CHECKING, Any
 
-from duckguard.core.result import ValidationResult
+from duckguard.core.result import FailedRow, ValidationResult
 
 if TYPE_CHECKING:
     from duckguard.core.dataset import Dataset
 
+# Default number of failed rows to capture for debugging
+DEFAULT_SAMPLE_SIZE = 10
+
 
 class Column:
     """
@@ -164,13 +166,14 @@ class Column:
             message=f"Column '{self._name}' unique_percent is {actual:.2f}% (threshold: {threshold}%)",
         )
 
-    def between(self, min_val: Any, max_val: Any) -> ValidationResult:
+    def between(self, min_val: Any, max_val: Any, capture_failures: bool = True) -> ValidationResult:
         """
         Check that all values are between min and max (inclusive).
 
         Args:
             min_val: Minimum allowed value
             max_val: Maximum allowed value
+            capture_failures: Whether to capture sample failing rows (default: True)
 
         Returns:
             ValidationResult indicating if all non-null values are in range
@@ -188,20 +191,53 @@ class Column:
         out_of_range = self._dataset.engine.fetch_value(sql) or 0
         passed = out_of_range == 0
 
+        # Capture sample of failing rows for debugging
+        failed_rows = []
+        if not passed and capture_failures:
+            failed_rows = self._get_failed_rows_between(min_val, max_val)
+
         return ValidationResult(
             passed=passed,
             actual_value=out_of_range,
             expected_value=0,
             message=f"Column '{self._name}' has {out_of_range} values outside [{min_val}, {max_val}]",
             details={"min": min_val, "max": max_val, "out_of_range_count": out_of_range},
+            failed_rows=failed_rows,
+            total_failures=out_of_range,
         )
 
-    def matches(self, pattern: str) -> ValidationResult:
+    def _get_failed_rows_between(self, min_val: Any, max_val: Any, limit: int = DEFAULT_SAMPLE_SIZE) -> list[FailedRow]:
+        """Get sample of rows that failed between check."""
+        ref = self._dataset.engine.get_source_reference(self._dataset.source)
+        col = f'"{self._name}"'
+
+        sql = f"""
+        SELECT row_number() OVER () as row_idx, {col} as val
+        FROM {ref}
+        WHERE {col} IS NOT NULL
+          AND ({col} < {min_val} OR {col} > {max_val})
+        LIMIT {limit}
+        """
+
+        rows = self._dataset.engine.fetch_all(sql)
+        return [
+            FailedRow(
+                row_index=row[0],
+                column=self._name,
+                value=row[1],
+                expected=f"between {min_val} and {max_val}",
+                reason=f"Value {row[1]} is outside range [{min_val}, {max_val}]",
+            )
+            for row in rows
+        ]
+
+    def matches(self, pattern: str, capture_failures: bool = True) -> ValidationResult:
         """
         Check that all non-null values match a regex pattern.
 
         Args:
             pattern: Regular expression pattern
+            capture_failures: Whether to capture sample failing rows (default: True)
 
         Returns:
             ValidationResult
@@ -220,20 +256,53 @@ class Column:
         non_matching = self._dataset.engine.fetch_value(sql) or 0
         passed = non_matching == 0
 
+        # Capture sample of failing rows
+        failed_rows = []
+        if not passed and capture_failures:
+            failed_rows = self._get_failed_rows_pattern(pattern)
+
         return ValidationResult(
             passed=passed,
             actual_value=non_matching,
             expected_value=0,
             message=f"Column '{self._name}' has {non_matching} values not matching pattern '{pattern}'",
             details={"pattern": pattern, "non_matching_count": non_matching},
+            failed_rows=failed_rows,
+            total_failures=non_matching,
         )
 
-    def isin(self, values: list[Any]) -> ValidationResult:
+    def _get_failed_rows_pattern(self, pattern: str, limit: int = DEFAULT_SAMPLE_SIZE) -> list[FailedRow]:
+        """Get sample of rows that failed pattern match."""
+        ref = self._dataset.engine.get_source_reference(self._dataset.source)
+        col = f'"{self._name}"'
+
+        sql = f"""
+        SELECT row_number() OVER () as row_idx, {col} as val
+        FROM {ref}
+        WHERE {col} IS NOT NULL
+          AND NOT regexp_matches({col}::VARCHAR, '{pattern}')
+        LIMIT {limit}
+        """
+
+        rows = self._dataset.engine.fetch_all(sql)
+        return [
+            FailedRow(
+                row_index=row[0],
+                column=self._name,
+                value=row[1],
+                expected=f"matches pattern '{pattern}'",
+                reason=f"Value '{row[1]}' does not match pattern",
+            )
+            for row in rows
+        ]
+
+    def isin(self, values: list[Any], capture_failures: bool = True) -> ValidationResult:
         """
         Check that all non-null values are in the allowed set.
 
         Args:
             values: List of allowed values
+            capture_failures: Whether to capture sample failing rows (default: True)
 
         Returns:
             ValidationResult
@@ -256,14 +325,50 @@ class Column:
         invalid_count = self._dataset.engine.fetch_value(sql) or 0
         passed = invalid_count == 0
 
+        # Capture sample of failing rows
+        failed_rows = []
+        if not passed and capture_failures:
+            failed_rows = self._get_failed_rows_isin(values)
+
         return ValidationResult(
             passed=passed,
             actual_value=invalid_count,
             expected_value=0,
             message=f"Column '{self._name}' has {invalid_count} values not in allowed set",
             details={"allowed_values": values, "invalid_count": invalid_count},
+            failed_rows=failed_rows,
+            total_failures=invalid_count,
         )
 
+    def _get_failed_rows_isin(self, values: list[Any], limit: int = DEFAULT_SAMPLE_SIZE) -> list[FailedRow]:
+        """Get sample of rows that failed isin check."""
+        ref = self._dataset.engine.get_source_reference(self._dataset.source)
+        col = f'"{self._name}"'
+
+        formatted_values = ", ".join(
+            f"'{v}'" if isinstance(v, str) else str(v) for v in values
+        )
+
+        sql = f"""
+        SELECT row_number() OVER () as row_idx, {col} as val
+        FROM {ref}
+        WHERE {col} IS NOT NULL
+          AND {col} NOT IN ({formatted_values})
+        LIMIT {limit}
+        """
+
+        rows = self._dataset.engine.fetch_all(sql)
+        return [
+            FailedRow(
+                row_index=row[0],
+                column=self._name,
+                value=row[1],
+                expected=f"in {values}",
+                reason=f"Value '{row[1]}' is not in allowed set",
+            )
+            for row in rows
+        ]
+
     def has_no_duplicates(self) -> ValidationResult:
         """
         Check that all values are unique (no duplicates).

duckguard/core/dataset.py

@@ -4,8 +4,8 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
-from duckguard.core.engine import DuckGuardEngine
 from duckguard.core.column import Column
+from duckguard.core.engine import DuckGuardEngine
 
 if TYPE_CHECKING:
     from duckguard.core.scoring import QualityScore
@@ -230,7 +230,7 @@ class Dataset:
     def score(
         self,
         weights: dict | None = None,
-    ) -> "QualityScore":
+    ) -> QualityScore:
         """
         Calculate data quality score for this dataset.
 
@@ -262,7 +262,7 @@ class Dataset:
                 'consistency': 0.1,
             })
         """
-        from duckguard.core.scoring import QualityScorer, QualityDimension
+        from duckguard.core.scoring import QualityDimension, QualityScorer
 
         # Convert string keys to QualityDimension enums if needed
         scorer_weights = None

duckguard/core/result.py

@@ -17,6 +17,30 @@ class CheckStatus(Enum):
     ERROR = "error"
 
 
+@dataclass
+class FailedRow:
+    """Represents a single row that failed validation.
+
+    Attributes:
+        row_index: The 1-based row number in the source data
+        column: The column name that failed validation
+        value: The actual value that failed
+        expected: What was expected (e.g., "not null", "between 1-100")
+        reason: Human-readable explanation of why validation failed
+        context: Additional row data for context (optional)
+    """
+
+    row_index: int
+    column: str
+    value: Any
+    expected: str
+    reason: str = ""
+    context: dict[str, Any] = field(default_factory=dict)
+
+    def __repr__(self) -> str:
+        return f"FailedRow(row={self.row_index}, column='{self.column}', value={self.value!r})"
+
+
 @dataclass
 class CheckResult:
     """Result of a single validation check."""
@@ -46,13 +70,27 @@ class CheckResult:
 
 @dataclass
 class ValidationResult:
-    """Result of a validation operation that can be used in assertions."""
+    """Result of a validation operation that can be used in assertions.
+
+    Enhanced with row-level error capture for debugging failed checks.
+
+    Attributes:
+        passed: Whether the validation passed
+        actual_value: The actual value found (e.g., count of failures)
+        expected_value: What was expected
+        message: Human-readable summary
+        details: Additional metadata
+        failed_rows: List of individual rows that failed validation
+        sample_size: How many failed rows to capture (default: 10)
+    """
 
     passed: bool
     actual_value: Any
     expected_value: Any | None = None
     message: str = ""
     details: dict[str, Any] = field(default_factory=dict)
+    failed_rows: list[FailedRow] = field(default_factory=list)
+    total_failures: int = 0
 
     def __bool__(self) -> bool:
         """Allow using ValidationResult in boolean context for assertions."""
@@ -60,8 +98,61 @@ class ValidationResult:
 
     def __repr__(self) -> str:
         status = "PASSED" if self.passed else "FAILED"
+        if self.failed_rows:
+            return f"ValidationResult({status}, actual={self.actual_value}, failed_rows={len(self.failed_rows)})"
         return f"ValidationResult({status}, actual={self.actual_value})"
 
+    def get_failed_values(self) -> list[Any]:
+        """Get list of values that failed validation."""
+        return [row.value for row in self.failed_rows]
+
+    def get_failed_row_indices(self) -> list[int]:
+        """Get list of row indices that failed validation."""
+        return [row.row_index for row in self.failed_rows]
+
+    def to_dataframe(self):
+        """Convert failed rows to a pandas DataFrame (if pandas available).
+
+        Returns:
+            pandas.DataFrame with failed row details
+
+        Raises:
+            ImportError: If pandas is not installed
+        """
+        try:
+            import pandas as pd
+
+            if not self.failed_rows:
+                return pd.DataFrame(columns=["row_index", "column", "value", "expected", "reason"])
+
+            return pd.DataFrame([
+                {
+                    "row_index": row.row_index,
+                    "column": row.column,
+                    "value": row.value,
+                    "expected": row.expected,
+                    "reason": row.reason,
+                    **row.context,
+                }
+                for row in self.failed_rows
+            ])
+        except ImportError:
+            raise ImportError("pandas is required for to_dataframe(). Install with: pip install pandas")
+
+    def summary(self) -> str:
+        """Get a summary of the validation result with sample failures."""
+        lines = [self.message]
+
+        if self.failed_rows:
+            lines.append(f"\nSample of {len(self.failed_rows)} failing rows (total: {self.total_failures}):")
+            for row in self.failed_rows[:5]:
+                lines.append(f"  Row {row.row_index}: {row.column}={row.value!r} - {row.reason or row.expected}")
+
+            if self.total_failures > 5:
+                lines.append(f"  ... and {self.total_failures - 5} more failures")
+
+        return "\n".join(lines)
+
 
 @dataclass
 class ProfileResult:

duckguard/core/scoring.py

@@ -14,7 +14,7 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from datetime import datetime
 from enum import Enum
-from typing import Any, TYPE_CHECKING
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from duckguard.core.dataset import Dataset
@@ -302,7 +302,6 @@ class QualityScorer:
         # Check for reasonable ranges on numeric columns
         if numeric_stats.get("mean") is not None:
             min_val = stats.get("min_value")
-            max_val = stats.get("max_value")
 
             # Check for negative values in likely positive-only columns
             is_likely_positive = any(