duckguard 2.0.0__py3-none-any.whl

duckguard/rules/schema.py

@@ -0,0 +1,289 @@
+"""Schema definitions for YAML-based rules.
+
+Defines the data structures that represent validation rules loaded from YAML.
+The schema is designed to be simple and readable, avoiding complex DSL syntax.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class CheckType(Enum):
+    """Types of validation checks."""
+
+    # Null checks
+    NOT_NULL = "not_null"
+    NULL_PERCENT = "null_percent"
+
+    # Uniqueness checks
+    UNIQUE = "unique"
+    UNIQUE_PERCENT = "unique_percent"
+    NO_DUPLICATES = "no_duplicates"
+
+    # Value checks
+    RANGE = "range"
+    BETWEEN = "between"
+    MIN = "min"
+    MAX = "max"
+    POSITIVE = "positive"
+    NEGATIVE = "negative"
+    NON_NEGATIVE = "non_negative"
+
+    # String checks
+    PATTERN = "pattern"
+    LENGTH = "length"
+    MIN_LENGTH = "min_length"
+    MAX_LENGTH = "max_length"
+
+    # Enum/Set checks
+    ALLOWED_VALUES = "allowed_values"
+    ISIN = "isin"
+    NOT_IN = "not_in"
+
+    # Type checks
+    TYPE = "type"
+    SEMANTIC_TYPE = "semantic_type"
+
+    # Statistical checks
+    MEAN = "mean"
+    STDDEV = "stddev"
+
+    # Anomaly checks
+    ANOMALY = "anomaly"
+
+    # Row-level checks
+    ROW_COUNT = "row_count"
+
+    # Custom SQL
+    CUSTOM_SQL = "custom_sql"
+
+
+class Severity(Enum):
+    """Severity levels for rule violations."""
+
+    ERROR = "error"      # Fails the check
+    WARNING = "warning"  # Reports but doesn't fail
+    INFO = "info"        # Informational only
+
+
+@dataclass
+class Check:
+    """A single validation check.
+
+    Attributes:
+        type: The type of check to perform
+        value: The expected value or threshold
+        operator: Comparison operator (=, <, >, <=, >=, !=)
+        severity: How severe a violation is
+        message: Custom message on failure
+        enabled: Whether the check is active
+    """
+
+    type: CheckType
+    value: Any = None
+    operator: str = "="
+    severity: Severity = Severity.ERROR
+    message: str | None = None
+    enabled: bool = True
+
+    # Additional parameters for complex checks
+    params: dict[str, Any] = field(default_factory=dict)
+
+    # Store the original column name for context
+    _column: str | None = field(default=None, repr=False)
+
+    def __post_init__(self):
+        # Convert string type to enum if needed
+        if isinstance(self.type, str):
+            self.type = CheckType(self.type)
+        if isinstance(self.severity, str):
+            self.severity = Severity(self.severity)
+
+    @property
+    def expression(self) -> str:
+        """Generate a human-readable expression for this check."""
+        col = self._column or ""
+
+        if self.type == CheckType.NOT_NULL:
+            return f"{col} is not null" if col else "is not null"
+        elif self.type == CheckType.UNIQUE:
+            return f"{col} is unique" if col else "is unique"
+        elif self.type == CheckType.NO_DUPLICATES:
+            return f"{col} has no duplicates" if col else "has no duplicates"
+        elif self.type == CheckType.ROW_COUNT:
+            return f"row_count {self.operator} {self.value}"
+        elif self.type == CheckType.NULL_PERCENT:
+            return f"{col} null_percent {self.operator} {self.value}" if col else f"null_percent {self.operator} {self.value}"
+        elif self.type == CheckType.UNIQUE_PERCENT:
+            return f"{col} unique_percent {self.operator} {self.value}" if col else f"unique_percent {self.operator} {self.value}"
+        elif self.type == CheckType.BETWEEN or self.type == CheckType.RANGE:
+            if isinstance(self.value, (list, tuple)) and len(self.value) == 2:
+                return f"{col} between {self.value[0]} and {self.value[1]}" if col else f"between {self.value[0]} and {self.value[1]}"
+        elif self.type == CheckType.MIN:
+            return f"{col} >= {self.value}" if col else f">= {self.value}"
+        elif self.type == CheckType.MAX:
+            return f"{col} <= {self.value}" if col else f"<= {self.value}"
+        elif self.type == CheckType.POSITIVE:
+            return f"{col} > 0" if col else "> 0"
+        elif self.type == CheckType.NEGATIVE:
+            return f"{col} < 0" if col else "< 0"
+        elif self.type == CheckType.NON_NEGATIVE:
+            return f"{col} >= 0" if col else ">= 0"
+        elif self.type == CheckType.PATTERN:
+            return f"{col} matches '{self.value}'" if col else f"matches '{self.value}'"
+        elif self.type == CheckType.ALLOWED_VALUES or self.type == CheckType.ISIN:
+            return f"{col} in {self.value}" if col else f"in {self.value}"
+
+        # Fallback
+        if col:
+            return f"{col} {self.type.value} {self.value}" if self.value else f"{col} {self.type.value}"
+        return f"{self.type.value} {self.value}" if self.value else self.type.value
+
+
+@dataclass
+class ColumnRules:
+    """Rules for a specific column.
+
+    Attributes:
+        name: Column name
+        checks: List of checks to apply
+        semantic_type: Detected or specified semantic type
+        description: Human-readable description
+        tags: Tags for grouping/filtering
+    """
+
+    name: str
+    checks: list[Check] = field(default_factory=list)
+    semantic_type: str | None = None
+    description: str | None = None
+    tags: list[str] = field(default_factory=list)
+
+
+@dataclass
+class TableRules:
+    """Table-level rules (row count, freshness, etc).
+
+    Attributes:
+        checks: List of table-level checks
+    """
+
+    checks: list[Check] = field(default_factory=list)
+
+
+@dataclass
+class SimpleCheck:
+    """A simple check with just an expression string.
+
+    Used for the simplified YAML rule syntax.
+    """
+    expression: str
+    column: str | None = None
+    check_type: CheckType | None = None
+    value: Any = None
+    operator: str = "="
+
+
+@dataclass
+class RuleSet:
+    """A complete set of validation rules for a data source.
+
+    Attributes:
+        source: Data source path or connection string
+        name: Human-readable name for this rule set
+        version: Version of the rule set
+        description: Description of what this validates
+        columns: Column-specific rules
+        table: Table-level rules
+        settings: Global settings for rule execution
+    """
+
+    source: str | None = None
+    name: str | None = None
+    version: str = "1.0"
+    description: str | None = None
+    columns: dict[str, ColumnRules] = field(default_factory=dict)
+    table: TableRules = field(default_factory=TableRules)
+    settings: dict[str, Any] = field(default_factory=dict)
+    # Simple rules list for the simplified format
+    _simple_checks: list[SimpleCheck] = field(default_factory=list)
+
+    @property
+    def dataset(self) -> str | None:
+        """Alias for name (for compatibility with simple syntax)."""
+        return self.name
+
+    @property
+    def checks(self) -> list[SimpleCheck]:
+        """Get all checks as a simple list."""
+        return self._simple_checks
+
+    def get_column_rules(self, column_name: str) -> ColumnRules | None:
+        """Get rules for a specific column."""
+        return self.columns.get(column_name)
+
+    def add_simple_check(self, expression: str) -> None:
+        """Add a simple check by expression string."""
+        self._simple_checks.append(SimpleCheck(expression=expression))
+
+    def add_column_check(
+        self,
+        column_name: str,
+        check_type: CheckType | str,
+        value: Any = None,
+        **kwargs
+    ) -> None:
+        """Add a check to a column."""
+        if column_name not in self.columns:
+            self.columns[column_name] = ColumnRules(name=column_name)
+
+        check = Check(
+            type=check_type if isinstance(check_type, CheckType) else CheckType(check_type),
+            value=value,
+            _column=column_name,
+            **kwargs
+        )
+        self.columns[column_name].checks.append(check)
+
+    def add_table_check(
+        self,
+        check_type: CheckType | str,
+        value: Any = None,
+        **kwargs
+    ) -> None:
+        """Add a table-level check."""
+        check = Check(
+            type=check_type if isinstance(check_type, CheckType) else CheckType(check_type),
+            value=value,
+            **kwargs
+        )
+        self.table.checks.append(check)
+
+    @property
+    def total_checks(self) -> int:
+        """Total number of checks in this rule set."""
+        column_checks = sum(len(col.checks) for col in self.columns.values())
+        table_checks = len(self.table.checks)
+        return column_checks + table_checks
+
+
+# Built-in patterns for common validations
+BUILTIN_PATTERNS = {
+    "email": r"^[\w\.\-\+]+@[\w\.\-]+\.[a-zA-Z]{2,}$",
+    "phone": r"^\+?[\d\s\-\(\)]{10,}$",
+    "uuid": r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$",
+    "url": r"^https?://[\w\.\-]+(/[\w\.\-\?=&%]*)?$",
+    "ip_address": r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$",
+    "ipv6": r"^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$",
+    "date_iso": r"^\d{4}-\d{2}-\d{2}$",
+    "datetime_iso": r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}",
+    "ssn": r"^\d{3}-\d{2}-\d{4}$",
+    "zip_us": r"^\d{5}(-\d{4})?$",
+    "credit_card": r"^\d{4}[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}$",
+    "slug": r"^[a-z0-9]+(?:-[a-z0-9]+)*$",
+    "alpha": r"^[a-zA-Z]+$",
+    "alphanumeric": r"^[a-zA-Z0-9]+$",
+    "numeric": r"^-?\d+\.?\d*$",
+}

duckguard/semantic/__init__.py

@@ -0,0 +1,31 @@
+"""Semantic type detection for DuckGuard.
+
+This module automatically detects the semantic meaning of data columns,
+such as email addresses, phone numbers, dates, currencies, and PII.
+
+Example:
+    from duckguard.semantic import detect_type, SemanticAnalyzer
+
+    analyzer = SemanticAnalyzer()
+    result = analyzer.analyze_column(column)
+    print(result.semantic_type)  # "email"
+    print(result.confidence)     # 0.95
+"""
+
+from duckguard.semantic.detector import (
+    SemanticType,
+    SemanticTypeResult,
+    detect_type,
+    detect_types_for_dataset,
+)
+from duckguard.semantic.analyzer import SemanticAnalyzer
+from duckguard.semantic.validators import get_validator_for_type
+
+__all__ = [
+    "SemanticType",
+    "SemanticTypeResult",
+    "detect_type",
+    "detect_types_for_dataset",
+    "SemanticAnalyzer",
+    "get_validator_for_type",
+]

duckguard/semantic/analyzer.py

@@ -0,0 +1,270 @@
+"""High-level semantic analyzer for DuckGuard.
+
+Provides comprehensive semantic analysis of datasets including
+type detection, PII identification, and validation suggestions.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from duckguard.core.dataset import Dataset
+from duckguard.semantic.detector import (
+    SemanticType,
+    SemanticTypeResult,
+    SemanticTypeDetector,
+    PII_TYPES,
+)
+
+
+@dataclass
+class ColumnAnalysis:
+    """Complete analysis of a single column.
+
+    Attributes:
+        name: Column name
+        semantic_type: Detected semantic type
+        confidence: Detection confidence
+        is_pii: Whether column contains PII
+        pii_warning: Warning message if PII detected
+        suggested_validations: Recommended validations
+        statistics: Column statistics
+    """
+
+    name: str
+    semantic_type: SemanticType
+    confidence: float
+    is_pii: bool = False
+    pii_warning: str | None = None
+    suggested_validations: list[str] = field(default_factory=list)
+    statistics: dict[str, Any] = field(default_factory=dict)
+    reasons: list[str] = field(default_factory=list)
+
+
+@dataclass
+class DatasetAnalysis:
+    """Complete semantic analysis of a dataset.
+
+    Attributes:
+        source: Data source path
+        row_count: Number of rows
+        column_count: Number of columns
+        columns: Analysis per column
+        pii_columns: List of columns containing PII
+        warnings: List of warnings
+    """
+
+    source: str
+    row_count: int
+    column_count: int
+    columns: list[ColumnAnalysis] = field(default_factory=list)
+    pii_columns: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    def get_column(self, name: str) -> ColumnAnalysis | None:
+        """Get analysis for a specific column."""
+        for col in self.columns:
+            if col.name == name:
+                return col
+        return None
+
+    @property
+    def has_pii(self) -> bool:
+        """Check if dataset contains any PII."""
+        return len(self.pii_columns) > 0
+
+    def get_validations_yaml(self) -> str:
+        """Generate YAML validation rules from analysis."""
+        lines = ["checks:"]
+
+        for col in self.columns:
+            if col.suggested_validations:
+                lines.append(f"  {col.name}:")
+                for validation in col.suggested_validations:
+                    lines.append(f"    - {validation}")
+
+        return "\n".join(lines)
+
+
+class SemanticAnalyzer:
+    """Analyzes datasets for semantic types and patterns."""
+
+    def __init__(self):
+        self._detector = SemanticTypeDetector()
+
+    def analyze(self, dataset: Dataset) -> DatasetAnalysis:
+        """Perform complete semantic analysis of a dataset.
+
+        Args:
+            dataset: Dataset to analyze
+
+        Returns:
+            DatasetAnalysis with all column analyses
+        """
+        analysis = DatasetAnalysis(
+            source=dataset.source,
+            row_count=dataset.row_count,
+            column_count=dataset.column_count,
+        )
+
+        for col_name in dataset.columns:
+            col_analysis = self.analyze_column(dataset, col_name)
+            analysis.columns.append(col_analysis)
+
+            if col_analysis.is_pii:
+                analysis.pii_columns.append(col_name)
+                analysis.warnings.append(
+                    f"⚠️ PII detected in column '{col_name}' ({col_analysis.semantic_type.value})"
+                )
+
+        return analysis
+
+    def analyze_column(self, dataset: Dataset, col_name: str) -> ColumnAnalysis:
+        """Analyze a single column.
+
+        Args:
+            dataset: Parent dataset
+            col_name: Column name to analyze
+
+        Returns:
+            ColumnAnalysis for the column
+        """
+        col = dataset[col_name]
+
+        # Get sample values
+        try:
+            sample_values = col.get_distinct_values(limit=100)
+        except Exception:
+            sample_values = []
+
+        # Detect semantic type
+        result = self._detector.detect(
+            col_name,
+            sample_values,
+            col.unique_percent,
+            col.null_percent,
+        )
+
+        # Build statistics
+        statistics = {
+            "null_count": col.null_count,
+            "null_percent": col.null_percent,
+            "unique_count": col.unique_count,
+            "unique_percent": col.unique_percent,
+            "total_count": col.total_count,
+        }
+
+        # Add numeric stats if available
+        try:
+            if col.mean is not None:
+                statistics["min"] = col.min
+                statistics["max"] = col.max
+                statistics["mean"] = col.mean
+        except Exception:
+            pass
+
+        # Generate PII warning
+        pii_warning = None
+        if result.is_pii:
+            pii_warning = self._generate_pii_warning(result.semantic_type)
+
+        return ColumnAnalysis(
+            name=col_name,
+            semantic_type=result.semantic_type,
+            confidence=result.confidence,
+            is_pii=result.is_pii,
+            pii_warning=pii_warning,
+            suggested_validations=result.suggested_validations,
+            statistics=statistics,
+            reasons=result.reasons,
+        )
+
+    def _generate_pii_warning(self, sem_type: SemanticType) -> str:
+        """Generate appropriate PII warning message."""
+        warnings = {
+            SemanticType.EMAIL: (
+                "Email addresses are PII. Consider: encryption at rest, "
+                "access controls, and GDPR compliance."
+            ),
+            SemanticType.PHONE: (
+                "Phone numbers are PII. Consider: encryption, "
+                "access controls, and regional privacy laws."
+            ),
+            SemanticType.SSN: (
+                "⚠️ CRITICAL: SSN is highly sensitive PII. "
+                "Requires encryption, strict access controls, "
+                "and compliance with data protection regulations."
+            ),
+            SemanticType.CREDIT_CARD: (
+                "⚠️ CRITICAL: Credit card numbers require PCI DSS compliance. "
+                "Must be encrypted and tokenized."
+            ),
+            SemanticType.PERSON_NAME: (
+                "Names are PII. Consider: purpose limitation, "
+                "consent requirements, and anonymization."
+            ),
+            SemanticType.ADDRESS: (
+                "Physical addresses are PII. Consider: "
+                "data minimization and access controls."
+            ),
+        }
+        return warnings.get(sem_type, "This column may contain personally identifiable information (PII).")
+
+    def quick_scan(self, dataset: Dataset) -> dict[str, SemanticType]:
+        """Quickly scan dataset and return type mapping.
+
+        Args:
+            dataset: Dataset to scan
+
+        Returns:
+            Dict mapping column names to semantic types
+        """
+        types = {}
+        for col_name in dataset.columns:
+            col = dataset[col_name]
+            try:
+                sample = col.get_distinct_values(limit=50)
+            except Exception:
+                sample = []
+
+            result = self._detector.detect(
+                col_name,
+                sample,
+                col.unique_percent,
+                col.null_percent,
+            )
+            types[col_name] = result.semantic_type
+
+        return types
+
+    def find_pii_columns(self, dataset: Dataset) -> list[tuple[str, SemanticType, str]]:
+        """Find all columns containing PII.
+
+        Args:
+            dataset: Dataset to scan
+
+        Returns:
+            List of (column_name, semantic_type, warning) tuples
+        """
+        pii_found = []
+
+        for col_name in dataset.columns:
+            col = dataset[col_name]
+            try:
+                sample = col.get_distinct_values(limit=50)
+            except Exception:
+                sample = []
+
+            result = self._detector.detect(
+                col_name,
+                sample,
+                col.unique_percent,
+                col.null_percent,
+            )
+
+            if result.is_pii:
+                warning = self._generate_pii_warning(result.semantic_type)
+                pii_found.append((col_name, result.semantic_type, warning))
+
+        return pii_found