kontra 0.5.2__py3-none-any.whl

kontra/scout/store.py

@@ -0,0 +1,208 @@
+# src/kontra/scout/store.py
+"""
+Profile storage for Kontra Scout.
+
+Stores scout profiles using the same backend infrastructure as validation state.
+Profiles are stored separately from validation states but can use the same
+storage backend (local, S3, PostgreSQL).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from kontra.version import VERSION
+from .types import DatasetProfile, ProfileState
+
+
+def fingerprint_source(source_uri: str) -> str:
+    """
+    Generate a stable fingerprint for a data source URI.
+
+    Args:
+        source_uri: The data source URI
+
+    Returns:
+        16-character hex fingerprint
+    """
+    # Normalize the URI
+    normalized = source_uri.strip()
+
+    # Hash it
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
+
+
+class LocalProfileStore:
+    """
+    Filesystem-based profile storage.
+
+    Stores profiles in .kontra/profiles/ directory:
+        .kontra/profiles/<source_fingerprint>/<timestamp>.json
+    """
+
+    def __init__(self, base_path: Optional[str] = None):
+        """
+        Initialize the local profile store.
+
+        Args:
+            base_path: Base directory for profile storage.
+                      Defaults to .kontra/profiles/ in cwd.
+        """
+        if base_path:
+            self.base_path = Path(base_path)
+        else:
+            self.base_path = Path.cwd() / ".kontra" / "profiles"
+
+    def _source_dir(self, source_fingerprint: str) -> Path:
+        """Get the directory for a source's profiles."""
+        return self.base_path / source_fingerprint
+
+    def _profile_filename(self, profiled_at: str) -> str:
+        """Generate filename from timestamp."""
+        # Use ISO format but replace : with - for filesystem compatibility
+        ts = profiled_at.replace(":", "-").replace("+", "_")
+        return f"{ts}.json"
+
+    def save(self, state: ProfileState) -> None:
+        """Save a profile state to the filesystem."""
+        source_dir = self._source_dir(state.source_fingerprint)
+        source_dir.mkdir(parents=True, exist_ok=True)
+
+        filename = self._profile_filename(state.profiled_at)
+        filepath = source_dir / filename
+
+        # Write atomically
+        temp_path = filepath.with_suffix(".tmp")
+        try:
+            temp_path.write_text(state.to_json(), encoding="utf-8")
+            temp_path.rename(filepath)
+        except Exception:
+            if temp_path.exists():
+                temp_path.unlink()
+            raise
+
+    def get_latest(self, source_fingerprint: str) -> Optional[ProfileState]:
+        """Get the most recent profile for a source."""
+        history = self.get_history(source_fingerprint, limit=1)
+        return history[0] if history else None
+
+    def get_history(
+        self,
+        source_fingerprint: str,
+        limit: int = 10,
+    ) -> List[ProfileState]:
+        """Get recent profile history for a source, newest first."""
+        source_dir = self._source_dir(source_fingerprint)
+
+        if not source_dir.exists():
+            return []
+
+        # List all JSON files
+        profile_files = sorted(
+            source_dir.glob("*.json"),
+            key=lambda p: p.name,
+            reverse=True,
+        )
+
+        states = []
+        for filepath in profile_files[:limit]:
+            try:
+                content = filepath.read_text(encoding="utf-8")
+                state = ProfileState.from_json(content)
+                states.append(state)
+            except Exception:
+                continue
+
+        return states
+
+    def list_sources(self) -> List[str]:
+        """List all source fingerprints with stored profiles."""
+        if not self.base_path.exists():
+            return []
+
+        sources = []
+        for item in self.base_path.iterdir():
+            if item.is_dir() and len(item.name) == 16:
+                sources.append(item.name)
+
+        return sorted(sources)
+
+    def clear(self, source_fingerprint: Optional[str] = None) -> int:
+        """Clear stored profiles."""
+        deleted = 0
+
+        if source_fingerprint:
+            source_dir = self._source_dir(source_fingerprint)
+            if source_dir.exists():
+                for filepath in source_dir.glob("*.json"):
+                    filepath.unlink()
+                    deleted += 1
+                try:
+                    source_dir.rmdir()
+                except OSError:
+                    pass
+        else:
+            if self.base_path.exists():
+                for source_dir in self.base_path.iterdir():
+                    if source_dir.is_dir():
+                        for filepath in source_dir.glob("*.json"):
+                            filepath.unlink()
+                            deleted += 1
+                        try:
+                            source_dir.rmdir()
+                        except OSError:
+                            pass
+
+        return deleted
+
+    def __repr__(self) -> str:
+        return f"LocalProfileStore(base_path={self.base_path})"
+
+
+def create_profile_state(profile: DatasetProfile) -> ProfileState:
+    """
+    Create a ProfileState from a DatasetProfile.
+
+    Args:
+        profile: The profiled dataset
+
+    Returns:
+        ProfileState ready for storage
+    """
+    return ProfileState(
+        source_fingerprint=fingerprint_source(profile.source_uri),
+        source_uri=profile.source_uri,
+        profiled_at=profile.profiled_at,
+        profile=profile,
+        engine_version=VERSION,
+    )
+
+
+# Default store
+_default_profile_store: Optional[LocalProfileStore] = None
+
+
+def get_default_profile_store() -> LocalProfileStore:
+    """Get the default profile store."""
+    global _default_profile_store
+    if _default_profile_store is None:
+        _default_profile_store = LocalProfileStore()
+    return _default_profile_store
+
+
+def get_profile_store(backend: str = "local") -> LocalProfileStore:
+    """
+    Get a profile store by backend identifier.
+
+    Currently only supports local storage. Future: S3, PostgreSQL.
+    """
+    if not backend or backend == "local":
+        return get_default_profile_store()
+
+    # For now, all backends use local profile storage
+    # Future: implement S3ProfileStore, PostgresProfileStore
+    return get_default_profile_store()

kontra/scout/suggest.py

@@ -0,0 +1,200 @@
+# src/kontra/scout/suggest.py
+"""
+Generate suggested validation rules based on profile analysis.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+from .patterns import get_pattern_regex
+from .types import DatasetProfile, ColumnProfile
+
+
+def generate_rules(profile: DatasetProfile) -> List[Dict[str, Any]]:
+    """
+    Generate suggested validation rules based on profile.
+
+    Returns a list of rule dictionaries in Kontra contract format.
+    """
+    rules: List[Dict[str, Any]] = []
+
+    for col in profile.columns:
+        col_rules = _suggest_column_rules(col)
+        rules.extend(col_rules)
+
+    return rules
+
+
+def generate_rules_yaml(profile: DatasetProfile) -> str:
+    """
+    Generate YAML-formatted contract with suggested rules.
+
+    Returns a complete contract YAML that can be used directly with `kontra validate`.
+    """
+    rules = generate_rules(profile)
+
+    if not rules:
+        return "# No rules suggested - dataset may be empty or all columns have high null rates\nrules: []"
+
+    lines = [
+        f"# Auto-generated contract from Kontra Scout",
+        f"# Source: {profile.source_uri}",
+        f"# Rows analyzed: {profile.row_count:,}",
+        f"# Generated by Kontra Scout v{profile.engine_version}",
+        "",
+        "name: suggested_contract",
+        f"description: Auto-generated from Scout profile",
+        "",
+        f'dataset: "{profile.source_uri}"',
+        "",
+        "rules:",
+    ]
+
+    # Add min_rows suggestion (90% of current count)
+    min_threshold = int(profile.row_count * 0.9)
+    if min_threshold > 0:
+        lines.append(f"  # Dataset: expecting at least {min_threshold:,} rows (90% of current)")
+        lines.append("  - name: min_rows")
+        lines.append(f"    params: {{ threshold: {min_threshold} }}")
+        lines.append("")
+
+    for rule in rules:
+        name = rule["name"]
+        params = rule.get("params", {})
+        comment = rule.get("_comment", "")
+
+        if comment:
+            lines.append(f"  # {comment}")
+
+        if params:
+            # Format params inline for simple cases, multi-line for complex
+            if len(params) <= 2 and all(
+                not isinstance(v, (list, dict)) or (isinstance(v, list) and len(v) <= 5)
+                for v in params.values()
+            ):
+                params_str = ", ".join(_format_param(k, v) for k, v in params.items())
+                lines.append(f"  - name: {name}")
+                lines.append(f"    params: {{ {params_str} }}")
+            else:
+                lines.append(f"  - name: {name}")
+                lines.append("    params:")
+                for k, v in params.items():
+                    lines.append(f"      {k}: {_format_value(v)}")
+        else:
+            lines.append(f"  - name: {name}")
+
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def _suggest_column_rules(col: ColumnProfile) -> List[Dict[str, Any]]:
+    """Generate rules for a single column based on its profile."""
+    rules: List[Dict[str, Any]] = []
+
+    # 1. not_null: Suggest if column has 0% nulls
+    if col.null_rate == 0 and col.row_count > 0:
+        rules.append({
+            "name": "not_null",
+            "params": {"column": col.name},
+            "_comment": f"{col.name}: 100% non-null",
+        })
+
+    # 2. unique: Suggest only for columns that look like identifiers
+    is_identifier = col.semantic_type == "identifier"
+    name_hints = any(hint in col.name.lower() for hint in ["_id", "id", "key", "uuid", "guid", "pk", "code"])
+    if col.uniqueness_ratio >= 0.999 and col.null_rate == 0 and (is_identifier or name_hints):
+        rules.append({
+            "name": "unique",
+            "params": {"column": col.name},
+            "_comment": f"{col.name}: 100% unique (primary key candidate)",
+        })
+
+    # 3. allowed_values: Suggest for low-cardinality columns with known values
+    if col.is_low_cardinality and col.values and len(col.values) <= 20:
+        rules.append({
+            "name": "allowed_values",
+            "params": {
+                "column": col.name,
+                "values": col.values,
+            },
+            "_comment": f"{col.name}: low cardinality ({len(col.values)} values)",
+        })
+
+    # 4. dtype: Suggest type validation
+    if col.dtype in ("int", "float", "bool", "date", "datetime", "string"):
+        # Map to Kontra/Polars type names
+        type_map = {
+            "int": "int64",
+            "float": "float64",
+            "bool": "bool",
+            "date": "date",
+            "datetime": "datetime",
+            "string": "utf8",
+        }
+        kontra_type = type_map.get(col.dtype, col.dtype)
+        rules.append({
+            "name": "dtype",
+            "params": {
+                "column": col.name,
+                "type": kontra_type,
+            },
+            "_comment": f"{col.name}: detected type {col.dtype_raw}",
+        })
+
+    # 5. regex: Suggest for detected patterns
+    for pattern in col.detected_patterns:
+        regex = get_pattern_regex(pattern)
+        if regex:
+            rules.append({
+                "name": "regex",
+                "params": {
+                    "column": col.name,
+                    "pattern": regex,
+                },
+                "_comment": f"{col.name}: detected {pattern} pattern",
+            })
+
+    # 6. freshness: Suggest for timestamp columns that look like update times
+    if col.dtype in ("datetime", "timestamp") or col.temporal is not None:
+        # Check if column name suggests it's an update/modified timestamp
+        name_lower = col.name.lower()
+        freshness_hints = ["updated", "modified", "timestamp", "created_at", "updated_at", "last_"]
+        if any(hint in name_lower for hint in freshness_hints):
+            rules.append({
+                "name": "freshness",
+                "params": {
+                    "column": col.name,
+                    "max_age": "7d",
+                },
+                "_comment": f"{col.name}: timestamp column, adjust max_age as needed",
+            })
+
+    return rules
+
+
+def _format_param(key: str, value: Any) -> str:
+    """Format a single parameter for inline YAML."""
+    return f"{key}: {_format_value(value)}"
+
+
+def _format_value(value: Any) -> str:
+    """Format a value for YAML output."""
+    if isinstance(value, str):
+        # Escape quotes and use quotes if needed
+        if any(c in value for c in ['"', "'", ":", "{", "}", "[", "]", ",", "\n"]):
+            escaped = value.replace("\\", "\\\\").replace('"', '\\"')
+            return f'"{escaped}"'
+        return f'"{value}"'
+    elif isinstance(value, bool):
+        return "true" if value else "false"
+    elif isinstance(value, (int, float)):
+        return str(value)
+    elif isinstance(value, list):
+        items = ", ".join(_format_value(v) for v in value)
+        return f"[{items}]"
+    elif value is None:
+        return "null"
+    else:
+        return str(value)