duckguard 2.0.0__py3-none-any.whl

duckguard/core/dataset.py

@@ -0,0 +1,284 @@
+"""Dataset class representing a data source for validation."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from duckguard.core.engine import DuckGuardEngine
+from duckguard.core.column import Column
+
+if TYPE_CHECKING:
+    from duckguard.core.scoring import QualityScore
+
+
+class Dataset:
+    """
+    Represents a data source with validation capabilities.
+
+    A Dataset wraps a data source (file, database table, etc.) and provides
+    a Pythonic interface for accessing columns and performing validations.
+
+    Example:
+        orders = Dataset("data/orders.csv")
+        assert orders.row_count > 0
+        assert orders.customer_id.null_percent < 5
+    """
+
+    def __init__(
+        self,
+        source: str,
+        engine: DuckGuardEngine | None = None,
+        name: str | None = None,
+    ):
+        """
+        Initialize a Dataset.
+
+        Args:
+            source: Path to file or connection string
+            engine: Optional DuckGuardEngine instance (uses singleton if not provided)
+            name: Optional name for the dataset (defaults to source)
+        """
+        self._source = source
+        self._engine = engine or DuckGuardEngine.get_instance()
+        self._name = name or source
+        self._columns_cache: list[str] | None = None
+        self._row_count_cache: int | None = None
+
+    @property
+    def source(self) -> str:
+        """Get the source path or connection string."""
+        return self._source
+
+    @property
+    def name(self) -> str:
+        """Get the dataset name."""
+        return self._name
+
+    @property
+    def engine(self) -> DuckGuardEngine:
+        """Get the underlying engine."""
+        return self._engine
+
+    @property
+    def row_count(self) -> int:
+        """
+        Get the number of rows in the dataset.
+
+        Returns:
+            Number of rows
+        """
+        if self._row_count_cache is None:
+            self._row_count_cache = self._engine.get_row_count(self._source)
+        return self._row_count_cache
+
+    @property
+    def columns(self) -> list[str]:
+        """
+        Get the list of column names.
+
+        Returns:
+            List of column names
+        """
+        if self._columns_cache is None:
+            self._columns_cache = self._engine.get_columns(self._source)
+        return self._columns_cache
+
+    @property
+    def column_count(self) -> int:
+        """Get the number of columns."""
+        return len(self.columns)
+
+    def __getattr__(self, name: str) -> Column:
+        """
+        Access columns as attributes.
+
+        This allows Pythonic access like: dataset.customer_id
+
+        Args:
+            name: Column name
+
+        Returns:
+            Column object for the specified column
+
+        Raises:
+            AttributeError: If the column doesn't exist
+        """
+        # Avoid infinite recursion for private attributes
+        if name.startswith("_"):
+            raise AttributeError(f"'{type(self).__name__}' has no attribute '{name}'")
+
+        # Check if column exists
+        if name not in self.columns:
+            raise AttributeError(
+                f"Column '{name}' not found. Available columns: {', '.join(self.columns)}"
+            )
+
+        return Column(name, self)
+
+    def __getitem__(self, name: str) -> Column:
+        """
+        Access columns using bracket notation.
+
+        This allows access like: dataset["customer_id"]
+
+        Args:
+            name: Column name
+
+        Returns:
+            Column object for the specified column
+        """
+        if name not in self.columns:
+            raise KeyError(
+                f"Column '{name}' not found. Available columns: {', '.join(self.columns)}"
+            )
+        return Column(name, self)
+
+    def column(self, name: str) -> Column:
+        """
+        Get a Column object by name.
+
+        Args:
+            name: Column name
+
+        Returns:
+            Column object
+        """
+        return self[name]
+
+    def has_column(self, name: str) -> bool:
+        """
+        Check if a column exists.
+
+        Args:
+            name: Column name to check
+
+        Returns:
+            True if column exists
+        """
+        return name in self.columns
+
+    def sample(self, n: int = 10) -> list[dict[str, Any]]:
+        """
+        Get a sample of rows from the dataset.
+
+        Args:
+            n: Number of rows to sample
+
+        Returns:
+            List of dictionaries representing rows
+        """
+        ref = self._engine.get_source_reference(self._source)
+        sql = f"SELECT * FROM {ref} LIMIT {n}"
+        result = self._engine.execute(sql)
+
+        columns = [desc[0] for desc in result.description]
+        rows = result.fetchall()
+
+        return [dict(zip(columns, row)) for row in rows]
+
+    def head(self, n: int = 5) -> list[dict[str, Any]]:
+        """
+        Get the first n rows from the dataset.
+
+        Args:
+            n: Number of rows
+
+        Returns:
+            List of dictionaries representing rows
+        """
+        return self.sample(n)
+
+    def execute_sql(self, sql: str) -> list[tuple[Any, ...]]:
+        """
+        Execute a custom SQL query against this dataset.
+
+        The query can reference the dataset using {source} placeholder.
+
+        Args:
+            sql: SQL query with optional {source} placeholder
+
+        Returns:
+            Query results as list of tuples
+        """
+        ref = self._engine.get_source_reference(self._source)
+        formatted_sql = sql.format(source=ref)
+        return self._engine.fetch_all(formatted_sql)
+
+    def clear_cache(self) -> None:
+        """Clear cached values (row count, columns)."""
+        self._row_count_cache = None
+        self._columns_cache = None
+
+    def __repr__(self) -> str:
+        return f"Dataset('{self._source}', rows={self.row_count}, columns={self.column_count})"
+
+    def __str__(self) -> str:
+        return f"Dataset: {self._name} ({self.row_count} rows, {self.column_count} columns)"
+
+    def __len__(self) -> int:
+        """Return the number of rows."""
+        return self.row_count
+
+    def __contains__(self, column: str) -> bool:
+        """Check if a column exists."""
+        return column in self.columns
+
+    def __iter__(self):
+        """Iterate over column names."""
+        return iter(self.columns)
+
+    def score(
+        self,
+        weights: dict | None = None,
+    ) -> "QualityScore":
+        """
+        Calculate data quality score for this dataset.
+
+        Evaluates data across standard quality dimensions:
+        - Completeness: Are all required values present?
+        - Uniqueness: Are values appropriately unique?
+        - Validity: Do values conform to expected formats/ranges?
+        - Consistency: Are values consistent?
+
+        Args:
+            weights: Optional custom weights for dimensions.
+                     Keys: 'completeness', 'uniqueness', 'validity', 'consistency'
+                     Values must sum to 1.0
+
+        Returns:
+            QualityScore with overall score, grade, and dimension breakdowns.
+
+        Example:
+            score = orders.score()
+            print(score.overall)        # 87.5
+            print(score.grade)          # 'B'
+            print(score.completeness)   # 95.0
+
+            # With custom weights
+            score = orders.score(weights={
+                'completeness': 0.4,
+                'uniqueness': 0.2,
+                'validity': 0.3,
+                'consistency': 0.1,
+            })
+        """
+        from duckguard.core.scoring import QualityScorer, QualityDimension
+
+        # Convert string keys to QualityDimension enums if needed
+        scorer_weights = None
+        if weights:
+            scorer_weights = {}
+            key_mapping = {
+                "completeness": QualityDimension.COMPLETENESS,
+                "uniqueness": QualityDimension.UNIQUENESS,
+                "validity": QualityDimension.VALIDITY,
+                "consistency": QualityDimension.CONSISTENCY,
+            }
+            for key, value in weights.items():
+                if isinstance(key, str):
+                    scorer_weights[key_mapping[key]] = value
+                else:
+                    scorer_weights[key] = value
+
+        scorer = QualityScorer(weights=scorer_weights)
+        return scorer.score(self)

duckguard/core/engine.py

@@ -0,0 +1,261 @@
+"""DuckDB-based execution engine for DuckGuard."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import duckdb
+
+
+class DuckGuardEngine:
+    """
+    Central DuckDB execution engine for DuckGuard.
+
+    This engine handles all database operations, providing a fast,
+    memory-efficient way to validate data from various sources.
+    """
+
+    _instance: DuckGuardEngine | None = None
+
+    def __init__(self, memory_limit: str | None = None):
+        """
+        Initialize the DuckGuard engine.
+
+        Args:
+            memory_limit: Optional memory limit for DuckDB (e.g., "4GB")
+        """
+        self.conn = duckdb.connect(":memory:")
+
+        # Configure DuckDB for optimal performance
+        # Wrap in try-except for compatibility with different DuckDB versions
+        try:
+            self.conn.execute("SET enable_progress_bar = false")
+        except duckdb.InvalidInputException:
+            # Setting not supported in this DuckDB version - ignore
+            pass
+
+        if memory_limit:
+            try:
+                self.conn.execute(f"SET memory_limit = '{memory_limit}'")
+            except duckdb.InvalidInputException:
+                pass
+
+        # Track registered sources
+        self._sources: dict[str, str] = {}
+
+    @classmethod
+    def get_instance(cls) -> DuckGuardEngine:
+        """Get or create the singleton engine instance."""
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    @classmethod
+    def reset_instance(cls) -> None:
+        """Reset the singleton instance (useful for testing)."""
+        if cls._instance is not None:
+            cls._instance.close()
+            cls._instance = None
+
+    def execute(self, sql: str, params: list[Any] | None = None) -> duckdb.DuckDBPyRelation:
+        """
+        Execute a SQL query and return the result.
+
+        Args:
+            sql: The SQL query to execute
+            params: Optional parameters for the query
+
+        Returns:
+            DuckDB relation with query results
+        """
+        if params:
+            return self.conn.execute(sql, params)
+        return self.conn.execute(sql)
+
+    def fetch_one(self, sql: str, params: list[Any] | None = None) -> tuple[Any, ...] | None:
+        """Execute a query and fetch one row."""
+        result = self.execute(sql, params)
+        return result.fetchone()
+
+    def fetch_all(self, sql: str, params: list[Any] | None = None) -> list[tuple[Any, ...]]:
+        """Execute a query and fetch all rows."""
+        result = self.execute(sql, params)
+        return result.fetchall()
+
+    def fetch_value(self, sql: str, params: list[Any] | None = None) -> Any:
+        """Execute a query and fetch a single value."""
+        row = self.fetch_one(sql, params)
+        return row[0] if row else None
+
+    def register_file(self, name: str, path: str) -> None:
+        """
+        Register a file as a named source.
+
+        Args:
+            name: Name to reference the source
+            path: Path to the file (CSV, Parquet, JSON)
+        """
+        # DuckDB auto-detects file type from extension
+        self._sources[name] = path
+
+    def register_dataframe(self, name: str, df: Any) -> None:
+        """
+        Register a DataFrame (pandas, polars, or pyarrow) as a named source.
+
+        Args:
+            name: Name to reference the source
+            df: DataFrame to register
+        """
+        self.conn.register(name, df)
+        self._sources[name] = f"registered:{name}"
+
+    def get_source_reference(self, name: str) -> str:
+        """
+        Get the SQL reference for a registered source.
+
+        Args:
+            name: Name of the registered source
+
+        Returns:
+            SQL-safe reference to the source
+        """
+        if name in self._sources:
+            source = self._sources[name]
+            if source.startswith("registered:"):
+                return name
+            # Return quoted path for file sources
+            return f"'{source}'"
+        # Assume it's a direct path or table name
+        return f"'{name}'" if "." in name or "/" in name or "\\" in name else name
+
+    def table_exists(self, name: str) -> bool:
+        """Check if a table or source exists."""
+        try:
+            self.execute(f"SELECT 1 FROM {self.get_source_reference(name)} LIMIT 1")
+            return True
+        except duckdb.Error:
+            return False
+
+    def get_columns(self, source: str) -> list[str]:
+        """
+        Get column names for a source.
+
+        Args:
+            source: Source reference (file path or registered name)
+
+        Returns:
+            List of column names
+        """
+        ref = self.get_source_reference(source)
+        result = self.execute(f"DESCRIBE SELECT * FROM {ref}")
+        return [row[0] for row in result.fetchall()]
+
+    def get_row_count(self, source: str) -> int:
+        """
+        Get row count for a source.
+
+        Args:
+            source: Source reference
+
+        Returns:
+            Number of rows
+        """
+        ref = self.get_source_reference(source)
+        return self.fetch_value(f"SELECT COUNT(*) FROM {ref}") or 0
+
+    def get_column_stats(self, source: str, column: str) -> dict[str, Any]:
+        """
+        Get basic statistics for a column.
+
+        Args:
+            source: Source reference
+            column: Column name
+
+        Returns:
+            Dictionary with column statistics
+        """
+        ref = self.get_source_reference(source)
+        col = f'"{column}"'
+
+        sql = f"""
+        SELECT
+            COUNT(*) as total_count,
+            COUNT({col}) as non_null_count,
+            COUNT(*) - COUNT({col}) as null_count,
+            COUNT(DISTINCT {col}) as unique_count,
+            MIN({col}) as min_value,
+            MAX({col}) as max_value
+        FROM {ref}
+        """
+
+        row = self.fetch_one(sql)
+        if not row:
+            return {}
+
+        total = row[0] or 0
+        non_null = row[1] or 0
+        null_count = row[2] or 0
+        unique_count = row[3] or 0
+
+        return {
+            "total_count": total,
+            "non_null_count": non_null,
+            "null_count": null_count,
+            "null_percent": (null_count / total * 100) if total > 0 else 0.0,
+            "unique_count": unique_count,
+            "unique_percent": (unique_count / total * 100) if total > 0 else 0.0,
+            "min_value": row[4],
+            "max_value": row[5],
+        }
+
+    def get_numeric_stats(self, source: str, column: str) -> dict[str, Any]:
+        """
+        Get numeric statistics for a column.
+
+        Args:
+            source: Source reference
+            column: Column name
+
+        Returns:
+            Dictionary with numeric statistics
+        """
+        ref = self.get_source_reference(source)
+        col = f'"{column}"'
+
+        sql = f"""
+        SELECT
+            AVG({col}::DOUBLE) as mean_value,
+            STDDEV({col}::DOUBLE) as stddev_value,
+            MEDIAN({col}::DOUBLE) as median_value,
+            PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY {col}::DOUBLE) as p25,
+            PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY {col}::DOUBLE) as p75
+        FROM {ref}
+        WHERE {col} IS NOT NULL
+        """
+
+        try:
+            row = self.fetch_one(sql)
+            if not row:
+                return {}
+
+            return {
+                "mean": row[0],
+                "stddev": row[1],
+                "median": row[2],
+                "p25": row[3],
+                "p75": row[4],
+            }
+        except duckdb.Error:
+            # Column might not be numeric
+            return {}
+
+    def close(self) -> None:
+        """Close the database connection."""
+        if self.conn:
+            self.conn.close()
+
+    def __enter__(self) -> DuckGuardEngine:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        self.close()

duckguard/core/result.py

@@ -0,0 +1,119 @@
+"""Result types for validation operations."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+
+class CheckStatus(Enum):
+    """Status of a validation check."""
+
+    PASSED = "passed"
+    FAILED = "failed"
+    WARNING = "warning"
+    ERROR = "error"
+
+
+@dataclass
+class CheckResult:
+    """Result of a single validation check."""
+
+    name: str
+    status: CheckStatus
+    actual_value: Any
+    expected_value: Any | None = None
+    message: str = ""
+    column: str | None = None
+    timestamp: datetime = field(default_factory=datetime.now)
+
+    @property
+    def passed(self) -> bool:
+        """Check if the validation passed."""
+        return self.status == CheckStatus.PASSED
+
+    @property
+    def failed(self) -> bool:
+        """Check if the validation failed."""
+        return self.status == CheckStatus.FAILED
+
+    def __bool__(self) -> bool:
+        """Allow using CheckResult in boolean context."""
+        return self.passed
+
+
+@dataclass
+class ValidationResult:
+    """Result of a validation operation that can be used in assertions."""
+
+    passed: bool
+    actual_value: Any
+    expected_value: Any | None = None
+    message: str = ""
+    details: dict[str, Any] = field(default_factory=dict)
+
+    def __bool__(self) -> bool:
+        """Allow using ValidationResult in boolean context for assertions."""
+        return self.passed
+
+    def __repr__(self) -> str:
+        status = "PASSED" if self.passed else "FAILED"
+        return f"ValidationResult({status}, actual={self.actual_value})"
+
+
+@dataclass
+class ProfileResult:
+    """Result of profiling a dataset."""
+
+    source: str
+    row_count: int
+    column_count: int
+    columns: list[ColumnProfile]
+    suggested_rules: list[str] = field(default_factory=list)
+    timestamp: datetime = field(default_factory=datetime.now)
+
+
+@dataclass
+class ColumnProfile:
+    """Profile information for a single column."""
+
+    name: str
+    dtype: str
+    null_count: int
+    null_percent: float
+    unique_count: int
+    unique_percent: float
+    min_value: Any | None = None
+    max_value: Any | None = None
+    mean_value: float | None = None
+    stddev_value: float | None = None
+    sample_values: list[Any] = field(default_factory=list)
+    suggested_rules: list[str] = field(default_factory=list)
+
+
+@dataclass
+class ScanResult:
+    """Result of scanning a dataset for issues."""
+
+    source: str
+    row_count: int
+    checks_run: int
+    checks_passed: int
+    checks_failed: int
+    checks_warned: int
+    results: list[CheckResult] = field(default_factory=list)
+    timestamp: datetime = field(default_factory=datetime.now)
+
+    @property
+    def passed(self) -> bool:
+        """Check if all validations passed."""
+        return self.checks_failed == 0
+
+    @property
+    def pass_rate(self) -> float:
+        """Calculate the pass rate as a percentage."""
+        if self.checks_run == 0:
+            return 100.0
+        return (self.checks_passed / self.checks_run) * 100