duckguard 3.0.1__py3-none-any.whl → 3.2.0__py3-none-any.whl

duckguard/__init__.py

@@ -85,7 +85,7 @@ from duckguard.semantic import (
     detect_types_for_dataset,
 )
 
-__version__ = "3.0.0"
+__version__ = "3.2.0"
 
 __all__ = [
     # Core classes

duckguard/ai/__init__.py

@@ -0,0 +1,33 @@
+"""AI-powered data quality features for DuckGuard.
+
+This module provides LLM-powered data quality capabilities:
+- explain: Natural language data quality summaries
+- suggest: AI-generated validation rules
+- fix: AI-suggested data cleaning steps
+- natural_rules: Plain English validation rules
+
+Requires: pip install duckguard[llm]
+
+Example:
+    from duckguard import connect
+    from duckguard.ai import explain, suggest_rules
+
+    orders = connect("orders.csv")
+    print(explain(orders))
+    rules = suggest_rules(orders)
+"""
+
+from duckguard.ai.config import configure, get_config
+from duckguard.ai.explainer import explain
+from duckguard.ai.fixer import suggest_fixes
+from duckguard.ai.natural_language import natural_rules
+from duckguard.ai.rules_generator import suggest_rules
+
+__all__ = [
+    "configure",
+    "get_config",
+    "explain",
+    "suggest_rules",
+    "suggest_fixes",
+    "natural_rules",
+]

duckguard/ai/config.py

@@ -0,0 +1,201 @@
+"""AI configuration for DuckGuard."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class AIConfig:
+    """Configuration for AI-powered features."""
+
+    provider: str = "openai"
+    model: str | None = None
+    api_key: str | None = None
+    base_url: str | None = None
+    temperature: float = 0.3
+    max_tokens: int = 2000
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def effective_model(self) -> str:
+        """Get the effective model name based on provider."""
+        if self.model:
+            return self.model
+        defaults = {
+            "openai": "gpt-4o-mini",
+            "anthropic": "claude-3-5-haiku-20241022",
+            "ollama": "llama3",
+        }
+        return defaults.get(self.provider, "gpt-4o-mini")
+
+    @property
+    def effective_api_key(self) -> str | None:
+        """Get API key from config or environment."""
+        if self.api_key:
+            return self.api_key
+        env_vars = {
+            "openai": "OPENAI_API_KEY",
+            "anthropic": "ANTHROPIC_API_KEY",
+        }
+        env_var = env_vars.get(self.provider)
+        if env_var:
+            return os.environ.get(env_var)
+        return None
+
+
+# Global config singleton
+_config: AIConfig | None = None
+
+
+def configure(
+    provider: str = "openai",
+    model: str | None = None,
+    api_key: str | None = None,
+    base_url: str | None = None,
+    temperature: float = 0.3,
+    **kwargs: Any,
+) -> AIConfig:
+    """
+    Configure the AI backend for DuckGuard.
+
+    Args:
+        provider: LLM provider ("openai", "anthropic", "ollama")
+        model: Model name (defaults based on provider)
+        api_key: API key (or set via environment variable)
+        base_url: Custom base URL (for Ollama or proxies)
+        temperature: Sampling temperature (default: 0.3 for consistency)
+        **kwargs: Additional provider-specific options
+
+    Returns:
+        AIConfig instance
+
+    Example:
+        from duckguard.ai import configure
+
+        # OpenAI
+        configure(provider="openai", api_key="sk-...")
+
+        # Anthropic
+        configure(provider="anthropic")  # uses ANTHROPIC_API_KEY env
+
+        # Local Ollama
+        configure(provider="ollama", model="llama3",
+                  base_url="http://localhost:11434")
+    """
+    global _config
+    _config = AIConfig(
+        provider=provider,
+        model=model,
+        api_key=api_key,
+        base_url=base_url,
+        temperature=temperature,
+        extra=kwargs,
+    )
+    return _config
+
+
+def get_config() -> AIConfig:
+    """Get the current AI configuration, or create a default one."""
+    global _config
+    if _config is None:
+        _config = AIConfig()
+    return _config
+
+
+def _get_client(config: AIConfig | None = None):
+    """
+    Get an LLM client based on configuration.
+
+    Returns a callable that takes a prompt and returns a string response.
+    """
+    cfg = config or get_config()
+
+    if cfg.provider == "openai":
+        try:
+            from openai import OpenAI
+        except ImportError:
+            raise ImportError(
+                "OpenAI support requires the openai package. "
+                "Install with: pip install duckguard[llm]"
+            )
+
+        client = OpenAI(
+            api_key=cfg.effective_api_key,
+            base_url=cfg.base_url,
+        )
+
+        def call_openai(prompt: str, system: str = "") -> str:
+            messages = []
+            if system:
+                messages.append({"role": "system", "content": system})
+            messages.append({"role": "user", "content": prompt})
+
+            response = client.chat.completions.create(
+                model=cfg.effective_model,
+                messages=messages,
+                temperature=cfg.temperature,
+                max_tokens=cfg.max_tokens,
+            )
+            return response.choices[0].message.content or ""
+
+        return call_openai
+
+    elif cfg.provider == "anthropic":
+        try:
+            from anthropic import Anthropic
+        except ImportError:
+            raise ImportError(
+                "Anthropic support requires the anthropic package. "
+                "Install with: pip install duckguard[llm]"
+            )
+
+        client = Anthropic(api_key=cfg.effective_api_key)
+
+        def call_anthropic(prompt: str, system: str = "") -> str:
+            response = client.messages.create(
+                model=cfg.effective_model,
+                max_tokens=cfg.max_tokens,
+                system=system if system else "You are a data quality expert.",
+                messages=[{"role": "user", "content": prompt}],
+            )
+            return response.content[0].text
+
+        return call_anthropic
+
+    elif cfg.provider == "ollama":
+        try:
+            from openai import OpenAI
+        except ImportError:
+            raise ImportError(
+                "Ollama support uses the openai package. "
+                "Install with: pip install openai"
+            )
+
+        client = OpenAI(
+            api_key="ollama",
+            base_url=cfg.base_url or "http://localhost:11434/v1",
+        )
+
+        def call_ollama(prompt: str, system: str = "") -> str:
+            messages = []
+            if system:
+                messages.append({"role": "system", "content": system})
+            messages.append({"role": "user", "content": prompt})
+
+            response = client.chat.completions.create(
+                model=cfg.effective_model,
+                messages=messages,
+                temperature=cfg.temperature,
+            )
+            return response.choices[0].message.content or ""
+
+        return call_ollama
+
+    else:
+        raise ValueError(
+            f"Unsupported AI provider: {cfg.provider}. "
+            f"Supported: openai, anthropic, ollama"
+        )

duckguard/ai/explainer.py

@@ -0,0 +1,109 @@
+"""AI-powered data quality explanation."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from duckguard.ai.config import _get_client
+
+if TYPE_CHECKING:
+    from duckguard.core.dataset import Dataset
+
+SYSTEM_PROMPT = """You are a data quality expert. You analyze dataset profiles and explain
+data quality issues in clear, actionable language. Be specific about which columns and
+values are problematic. Suggest concrete validation rules using DuckGuard's API.
+
+DuckGuard API methods:
+- column.is_not_null() — check for nulls
+- column.is_unique() — check uniqueness
+- column.between(min, max) — range check
+- column.isin(values) — enum check
+- column.matches(pattern) — regex pattern
+- column.not_null_when(condition) — conditional not-null
+- column.between_when(min, max, condition) — conditional range
+- dataset.score() — quality score (A-F)
+- detect_anomalies(dataset) — anomaly detection
+
+Keep explanations concise and actionable. Use emoji for visual clarity."""
+
+
+def explain(
+    dataset: Dataset,
+    focus: str | None = None,
+    detail: str = "medium",
+) -> str:
+    """
+    Generate a natural language explanation of data quality.
+
+    Args:
+        dataset: Dataset to analyze
+        focus: Optional column or aspect to focus on
+        detail: Level of detail ("brief", "medium", "detailed")
+
+    Returns:
+        Human-readable data quality explanation
+
+    Example:
+        from duckguard import connect
+        from duckguard.ai import explain
+
+        orders = connect("orders.csv")
+        print(explain(orders))
+    """
+    from duckguard.profiler import AutoProfiler
+
+    # Profile the dataset
+    profiler = AutoProfiler(deep=True)
+    profile = profiler.profile(dataset)
+
+    # Build context for the LLM
+    context_parts = [
+        f"Dataset: {dataset.name}",
+        f"Rows: {profile.row_count}, Columns: {profile.column_count}",
+        f"Overall Quality: {profile.overall_quality_grade} ({profile.overall_quality_score:.1f}/100)",
+        "",
+        "Column Profiles:",
+    ]
+
+    for col in profile.columns:
+        col_info = f"  {col.name} ({col.dtype}): "
+        col_info += f"nulls={col.null_percent:.1f}%, "
+        col_info += f"unique={col.unique_percent:.1f}%, "
+        col_info += f"grade={col.quality_grade}"
+
+        if col.min_value is not None:
+            col_info += f", range=[{col.min_value}, {col.max_value}]"
+        if col.detected_patterns:
+            col_info += f", patterns={col.detected_patterns}"
+        if col.distribution_type:
+            col_info += f", dist={col.distribution_type}"
+        if col.outlier_count and col.outlier_count > 0:
+            col_info += f", outliers={col.outlier_count}"
+
+        context_parts.append(col_info)
+
+    if profile.suggested_rules:
+        context_parts.append("")
+        context_parts.append(f"Auto-suggested rules ({len(profile.suggested_rules)}):")
+        for rule in profile.suggested_rules[:10]:
+            context_parts.append(f"  - {rule}")
+
+    context = "\n".join(context_parts)
+
+    # Build prompt
+    detail_instruction = {
+        "brief": "Give a 3-5 sentence summary.",
+        "medium": "Give a comprehensive but concise analysis (10-15 lines).",
+        "detailed": "Give a thorough analysis with specific recommendations.",
+    }.get(detail, "Give a comprehensive but concise analysis.")
+
+    focus_instruction = f"\nFocus specifically on: {focus}" if focus else ""
+
+    prompt = f"""Analyze this dataset profile and explain the data quality status.
+{detail_instruction}{focus_instruction}
+
+{context}"""
+
+    # Call LLM
+    client = _get_client()
+    return client(prompt, system=SYSTEM_PROMPT)

duckguard/ai/fixer.py

@@ -0,0 +1,105 @@
+"""AI-powered data fix suggestions."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from duckguard.ai.config import _get_client
+
+if TYPE_CHECKING:
+    from duckguard.core.dataset import Dataset
+
+SYSTEM_PROMPT = """You are a data quality expert. Given a dataset profile with quality issues,
+suggest specific fixes. For each issue:
+
+1. Describe the problem clearly
+2. Assess severity (critical / warning / info)
+3. Suggest a concrete fix (SQL, Python code, or process change)
+4. Note if no action is needed (e.g., nulls are expected for pending orders)
+
+Be practical. Not every null is a bug. Use context to determine what's actually wrong.
+Format with emoji and clear sections."""
+
+
+def suggest_fixes(
+    dataset: Dataset,
+    rules_result=None,
+) -> str:
+    """
+    Get AI-suggested fixes for data quality issues.
+
+    Args:
+        dataset: Dataset to analyze
+        rules_result: Optional RuleExecutionResult from a previous validation run
+
+    Returns:
+        Human-readable fix suggestions
+
+    Example:
+        from duckguard import connect
+        from duckguard.ai import suggest_fixes
+
+        orders = connect("orders.csv")
+        print(suggest_fixes(orders))
+    """
+    from duckguard.profiler import AutoProfiler
+
+    # Profile the dataset
+    profiler = AutoProfiler(deep=True)
+    profile = profiler.profile(dataset)
+
+    # Build context
+    context_parts = [
+        f"Dataset: {dataset.name} ({profile.row_count} rows, {profile.column_count} columns)",
+        f"Quality: {profile.overall_quality_grade} ({profile.overall_quality_score:.1f}/100)",
+        "",
+        "Issues detected:",
+    ]
+
+    has_issues = False
+
+    for col in profile.columns:
+        issues = []
+
+        if col.null_percent > 0:
+            issues.append(f"nulls: {col.null_percent:.1f}% ({col.null_count} rows)")
+
+        if col.quality_grade in ("D", "F"):
+            issues.append(f"low quality grade: {col.quality_grade}")
+
+        if col.outlier_count and col.outlier_count > 0:
+            issues.append(f"outliers: {col.outlier_count} ({col.outlier_percentage:.1f}%)")
+
+        if issues:
+            has_issues = True
+            context_parts.append(f"  {col.name} ({col.dtype}): {'; '.join(issues)}")
+
+            # Add sample values for context
+            if col.min_value is not None:
+                context_parts.append(f"    range: [{col.min_value}, {col.max_value}]")
+
+    if not has_issues:
+        return "✅ No data quality issues detected. Your data looks clean!"
+
+    # Add validation results if provided
+    if rules_result:
+        context_parts.append("")
+        context_parts.append("Failed validation checks:")
+        for r in getattr(rules_result, "results", []):
+            if not r.passed:
+                context_parts.append(f"  ✗ {r.message}")
+
+    context = "\n".join(context_parts)
+
+    prompt = f"""Analyze these data quality issues and suggest specific fixes.
+
+{context}
+
+For each issue, provide:
+1. What's wrong (brief)
+2. Severity (🔴 critical / 🟡 warning / 🔵 info)
+3. Suggested fix (code or process)
+4. Whether it actually needs fixing (sometimes nulls are expected)"""
+
+    client = _get_client()
+    return client(prompt, system=SYSTEM_PROMPT)

duckguard/ai/natural_language.py

@@ -0,0 +1,119 @@
+"""Natural language validation rules."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from duckguard.ai.config import _get_client
+from duckguard.core.result import ValidationResult
+
+if TYPE_CHECKING:
+    from duckguard.core.dataset import Dataset
+
+SYSTEM_PROMPT = """You are a data quality expert. Convert natural language rules into
+DuckGuard Python code.
+
+Available DuckGuard methods:
+- dataset.column_name.is_not_null()
+- dataset.column_name.is_unique()
+- dataset.column_name.between(min, max)
+- dataset.column_name.greater_than(value)
+- dataset.column_name.less_than(value)
+- dataset.column_name.isin(values_list)
+- dataset.column_name.matches(regex_pattern)
+- dataset.column_name.value_lengths_between(min, max)
+- dataset.column_name.not_null_when(sql_condition)
+- dataset.column_name.between_when(min, max, sql_condition)
+- dataset.column_name.exists_in(other_dataset.column)
+- dataset.expect_columns_unique(column_list)
+
+Dataset columns: {columns}
+
+For each natural language rule, output ONLY a Python expression that calls the appropriate
+DuckGuard method. One expression per line. No explanations, no imports.
+
+Example input: "order IDs should never be null"
+Example output: dataset.order_id.is_not_null()
+
+Example input: "quantities between 1 and 1000"
+Example output: dataset.quantity.between(1, 1000)"""
+
+
+def natural_rules(
+    dataset: Dataset,
+    rules: list[str],
+) -> list[ValidationResult]:
+    """
+    Validate data using natural language rules.
+
+    Converts plain English rules into DuckGuard validations and executes them.
+
+    Args:
+        dataset: Dataset to validate
+        rules: List of natural language rule descriptions
+
+    Returns:
+        List of ValidationResult objects
+
+    Example:
+        from duckguard import connect
+        from duckguard.ai import natural_rules
+
+        orders = connect("orders.csv")
+        results = natural_rules(orders, [
+            "order IDs should never be null or duplicated",
+            "quantities should be positive integers under 1000",
+            "status must be pending, shipped, or delivered",
+        ])
+
+        for r in results:
+            print(f"{'✓' if r.passed else '✗'} {r.message}")
+    """
+    columns = dataset.columns
+
+    # Build prompt with all rules
+    rules_text = "\n".join(f"Rule {i+1}: {rule}" for i, rule in enumerate(rules))
+
+    system = SYSTEM_PROMPT.format(columns=columns)
+    prompt = f"""Convert these natural language rules to DuckGuard expressions:
+
+{rules_text}
+
+Output one DuckGuard expression per rule, numbered to match. Use 'dataset' as the variable name."""
+
+    client = _get_client()
+    response = client(prompt, system=system)
+
+    # Parse and execute the generated expressions
+    results = []
+    expressions = [line.strip() for line in response.strip().split("\n") if line.strip()]
+
+    for i, expr in enumerate(expressions):
+        # Clean up the expression
+        expr = expr.lstrip("0123456789.:)-— ")
+        if not expr.startswith("dataset."):
+            continue
+
+        try:
+            # Execute the expression safely
+            result = eval(expr, {"dataset": dataset, "__builtins__": {}})  # noqa: S307
+            if isinstance(result, ValidationResult):
+                results.append(result)
+            elif isinstance(result, bool):
+                rule_desc = rules[i] if i < len(rules) else expr
+                results.append(ValidationResult(
+                    passed=result,
+                    actual_value=result,
+                    expected_value=True,
+                    message=f"Natural rule: {rule_desc}",
+                ))
+        except Exception as e:
+            rule_desc = rules[i] if i < len(rules) else expr
+            results.append(ValidationResult(
+                passed=False,
+                actual_value=str(e),
+                expected_value="valid expression",
+                message=f"Failed to evaluate rule '{rule_desc}': {e}",
+            ))
+
+    return results

duckguard/ai/rules_generator.py

@@ -0,0 +1,121 @@
+"""AI-powered validation rule generation."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from duckguard.ai.config import _get_client
+
+if TYPE_CHECKING:
+    from duckguard.core.dataset import Dataset
+
+SYSTEM_PROMPT = """You are a data quality expert. Generate DuckGuard YAML validation rules
+based on dataset profiles. Rules should be practical, not overly strict.
+
+Output format — valid YAML only, no markdown fences:
+
+name: <dataset>_validation
+description: Auto-generated quality checks
+
+checks:
+  column_name:
+    - not_null
+    - unique
+    - between: [min, max]
+    - allowed_values: [val1, val2]
+    - pattern: "regex"
+
+Rules to consider:
+- not_null for columns with 0% nulls (they're probably required)
+- unique for ID-like columns (>99% unique)
+- between for numeric columns (use actual range with small buffer)
+- allowed_values for low-cardinality columns (<20 distinct values)
+- pattern for columns matching known patterns (email, phone, etc.)
+
+Be conservative — generate rules that reflect the actual data, not hypothetical constraints.
+Only output the YAML. No explanations."""
+
+
+def suggest_rules(
+    dataset: Dataset,
+    strict: bool = False,
+    include_comments: bool = True,
+) -> str:
+    """
+    Generate validation rules using AI analysis.
+
+    Combines DuckGuard's profiling with LLM intelligence to generate
+    context-aware YAML rules that match your data's actual patterns.
+
+    Args:
+        dataset: Dataset to analyze
+        strict: If True, generate stricter rules
+        include_comments: If True, add explanatory comments
+
+    Returns:
+        YAML string with validation rules
+
+    Example:
+        from duckguard import connect
+        from duckguard.ai import suggest_rules
+
+        orders = connect("orders.csv")
+        yaml_rules = suggest_rules(orders)
+        print(yaml_rules)
+
+        # Save to file
+        with open("duckguard.yaml", "w") as f:
+            f.write(yaml_rules)
+    """
+    from duckguard.profiler import AutoProfiler
+
+    # Profile the dataset
+    profiler = AutoProfiler()
+    profile = profiler.profile(dataset)
+
+    # Build context
+    context_parts = [
+        f"Dataset: {dataset.name}",
+        f"Rows: {profile.row_count}",
+        "",
+        "Columns:",
+    ]
+
+    for col in profile.columns:
+        col_info = f"  {col.name}:"
+        col_info += f" type={col.dtype},"
+        col_info += f" nulls={col.null_percent:.1f}%,"
+        col_info += f" unique={col.unique_percent:.1f}%,"
+        col_info += f" distinct={col.unique_count}"
+
+        if col.min_value is not None:
+            col_info += f", min={col.min_value}, max={col.max_value}"
+        if col.detected_patterns:
+            col_info += f", patterns={col.detected_patterns}"
+        if hasattr(col, "sample_values") and col.sample_values:
+            col_info += f", samples={col.sample_values[:5]}"
+
+        context_parts.append(col_info)
+
+    # Include auto-detected rules as baseline
+    if profile.suggested_rules:
+        context_parts.append("")
+        context_parts.append("Auto-detected rules (baseline):")
+        for rule in profile.suggested_rules:
+            context_parts.append(f"  - {rule}")
+
+    context = "\n".join(context_parts)
+
+    strictness = "Generate strict rules — flag anything suspicious." if strict else \
+        "Generate practical rules — match actual data patterns, allow reasonable variation."
+
+    comment_instruction = "Add YAML comments explaining each rule." if include_comments else \
+        "No comments, just rules."
+
+    prompt = f"""{strictness}
+{comment_instruction}
+
+{context}"""
+
+    client = _get_client()
+    return client(prompt, system=SYSTEM_PROMPT)