data-contract-validator 1.0.0__py3-none-any.whl

data_contract_validator/core/models.py

@@ -0,0 +1,115 @@
+"""
+Core data models for validation results and schema definitions.
+"""
+
+from enum import Enum
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Any
+
+
+class IssueSeverity(Enum):
+    """Severity levels for validation issues."""
+
+    CRITICAL = "critical"  # Will break API
+    WARNING = "warning"  # Might cause issues
+    INFO = "info"  # Good to know
+
+
+@dataclass
+class ValidationIssue:
+    """Represents a single validation issue."""
+
+    severity: IssueSeverity
+    table: str
+    column: Optional[str]
+    message: str
+    category: str = "Unknown"
+    suggested_fix: Optional[str] = None
+    source_value: Optional[str] = None
+    target_value: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "severity": self.severity.value,
+            "table": self.table,
+            "column": self.column,
+            "message": self.message,
+            "category": self.category,
+            "suggested_fix": self.suggested_fix,
+            "source_value": self.source_value,
+            "target_value": self.target_value,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ValidationIssue":
+        """Create from dictionary."""
+        return cls(
+            severity=IssueSeverity(data.get("severity", "warning")),
+            table=data.get("table", "Unknown"),
+            column=data.get("column"),
+            message=data.get("message", ""),
+            category=data.get("category", "Unknown"),
+            suggested_fix=data.get("suggested_fix"),
+            source_value=data.get("source_value"),
+            target_value=data.get("target_value"),
+        )
+
+
+@dataclass
+class Schema:
+    """Represents a table schema."""
+
+    name: str
+    columns: List[Dict[str, Any]]
+    source: str = "unknown"
+    metadata: Optional[Dict[str, Any]] = None
+
+    def get_column(self, name: str) -> Optional[Dict[str, Any]]:
+        """Get column by name."""
+        for col in self.columns:
+            if col.get("name") == name:
+                return col
+        return None
+
+    def column_names(self) -> List[str]:
+        """Get list of column names."""
+        return [col.get("name") for col in self.columns if col.get("name")]
+
+
+@dataclass
+class ValidationResult:
+    """Result of contract validation."""
+
+    success: bool
+    issues: List[ValidationIssue]
+    source_schemas: Dict[str, Schema]
+    target_schemas: Dict[str, Schema]
+    summary: Optional[str] = None
+
+    @property
+    def critical_issues(self) -> List[ValidationIssue]:
+        """Get only critical issues."""
+        return [i for i in self.issues if i.severity == IssueSeverity.CRITICAL]
+
+    @property
+    def warnings(self) -> List[ValidationIssue]:
+        """Get only warnings."""
+        return [i for i in self.issues if i.severity == IssueSeverity.WARNING]
+
+    @property
+    def info_items(self) -> List[ValidationIssue]:
+        """Get only info items."""
+        return [i for i in self.issues if i.severity == IssueSeverity.INFO]
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "success": self.success,
+            "summary": self.summary,
+            "total_issues": len(self.issues),
+            "critical_issues": len(self.critical_issues),
+            "warnings": len(self.warnings),
+            "info_items": len(self.info_items),
+            "issues": [issue.to_dict() for issue in self.issues],
+        }

data_contract_validator/core/validator.py

@@ -0,0 +1,187 @@
+"""
+Core validation logic for comparing schemas.
+"""
+
+from typing import Dict, List
+from .models import ValidationResult, ValidationIssue, IssueSeverity, Schema
+from ..extractors.base import BaseExtractor
+
+
+class ContractValidator:
+    """
+    Main contract validator that compares schemas from different sources.
+    """
+
+    def __init__(
+        self, source_extractor: BaseExtractor, target_extractor: BaseExtractor
+    ):
+        """
+        Initialize validator with source and target extractors.
+
+        Args:
+            source_extractor: Extractor for source schemas (e.g., DBT)
+            target_extractor: Extractor for target schemas (e.g., FastAPI)
+        """
+        self.source_extractor = source_extractor
+        self.target_extractor = target_extractor
+        self.issues: List[ValidationIssue] = []
+
+    def validate(self) -> ValidationResult:
+        """
+        Run validation and return results.
+
+        Returns:
+            ValidationResult with success status and any issues found
+        """
+        print("🔍 Starting contract validation...")
+
+        # Extract schemas
+        print("📊 Extracting source schemas...")
+        source_schemas = self.source_extractor.extract_schemas()
+
+        print("🎯 Extracting target schemas...")
+        target_schemas = self.target_extractor.extract_schemas()
+
+        print(f"   Source: {len(source_schemas)} schemas")
+        print(f"   Target: {len(target_schemas)} schemas")
+
+        # Reset issues
+        self.issues = []
+
+        # Validate each target schema against source
+        print("🔍 Validating schema compatibility...")
+        for table_name, target_schema in target_schemas.items():
+            self._validate_table(table_name, target_schema, source_schemas)
+
+        # Determine success
+        critical_issues = [
+            i for i in self.issues if i.severity == IssueSeverity.CRITICAL
+        ]
+        success = len(critical_issues) == 0
+
+        # Generate summary
+        summary = self._generate_summary(success, self.issues)
+
+        return ValidationResult(
+            success=success,
+            issues=self.issues,
+            source_schemas=source_schemas,
+            target_schemas=target_schemas,
+            summary=summary,
+        )
+
+    def _validate_table(
+        self, table_name: str, target_schema: Schema, source_schemas: Dict[str, Schema]
+    ):
+        """Validate a single table."""
+        print(f"  🔍 Validating table: {table_name}")
+
+        # Check if source provides this table
+        source_schema = source_schemas.get(table_name)
+        if not source_schema:
+            self.issues.append(
+                ValidationIssue(
+                    severity=IssueSeverity.CRITICAL,
+                    table=table_name,
+                    column=None,
+                    message=f"Target expects table '{table_name}' but source doesn't provide it",
+                    category="Missing Table",
+                    suggested_fix=f"Create a source model that outputs table '{table_name}'",
+                )
+            )
+            print(f"    ❌ Table '{table_name}' missing in source")
+            return
+
+        # Check columns
+        source_columns = {col["name"]: col for col in source_schema.columns}
+        target_columns = {col["name"]: col for col in target_schema.columns}
+
+        # Check for missing required columns
+        for col_name, col_info in target_columns.items():
+            if col_name not in source_columns:
+                is_required = col_info.get("required", True)
+                severity = (
+                    IssueSeverity.CRITICAL if is_required else IssueSeverity.WARNING
+                )
+
+                self.issues.append(
+                    ValidationIssue(
+                        severity=severity,
+                        table=table_name,
+                        column=col_name,
+                        message=f"Target {'REQUIRES' if is_required else 'expects'} column '{col_name}' but source doesn't provide it",
+                        category="Missing Column",
+                        suggested_fix=f"Add column '{col_name}' to source model for table '{table_name}'",
+                    )
+                )
+            else:
+                # Check type compatibility
+                source_col = source_columns[col_name]
+                target_col = col_info
+
+                if not self._types_compatible(
+                    source_col.get("type"), target_col.get("type")
+                ):
+                    self.issues.append(
+                        ValidationIssue(
+                            severity=IssueSeverity.WARNING,
+                            table=table_name,
+                            column=col_name,
+                            message=f"Type mismatch: source provides '{source_col.get('type')}' but target expects '{target_col.get('type')}'",
+                            category="Type Mismatch",
+                            source_value=source_col.get("type"),
+                            target_value=target_col.get("type"),
+                            suggested_fix=f"Update target model to expect '{source_col.get('type')}' or fix source column type",
+                        )
+                    )
+
+        # Log results for this table
+        table_issues = [i for i in self.issues if i.table == table_name]
+        if not table_issues:
+            print(f"    ✅ All requirements satisfied")
+        else:
+            critical = [i for i in table_issues if i.severity == IssueSeverity.CRITICAL]
+            warnings = [i for i in table_issues if i.severity == IssueSeverity.WARNING]
+            if critical:
+                print(f"    🚨 {len(critical)} critical issues")
+            if warnings:
+                print(f"    ⚠️  {len(warnings)} warnings")
+
+    def _types_compatible(self, source_type: str, target_type: str) -> bool:
+        """Check if source and target types are compatible."""
+        if not source_type or not target_type:
+            return True  # Skip validation if types are unknown
+
+        # Normalize types
+        source_type = source_type.lower()
+        target_type = target_type.lower()
+
+        # Exact match
+        if source_type == target_type:
+            return True
+
+        # Compatible type mappings
+        compatible_types = {
+            "varchar": ["string", "str", "text"],
+            "string": ["varchar", "text"],
+            "text": ["varchar", "string"],
+            "integer": ["int", "bigint"],
+            "int": ["integer", "bigint"],
+            "bigint": ["integer", "int"],
+            "float": ["double", "decimal", "numeric", "real"],
+            "double": ["float", "decimal"],
+            "boolean": ["bool"],
+            "bool": ["boolean"],
+            "timestamp": ["datetime"],
+            "datetime": ["timestamp"],
+        }
+
+        return target_type in compatible_types.get(source_type, [])
+
+    def _generate_summary(self, success: bool, issues: List[ValidationIssue]) -> str:
+        """Generate validation summary."""
+        if success:
+            return f"✅ Validation passed with {len(issues)} non-critical issues"
+        else:
+            critical = [i for i in issues if i.severity == IssueSeverity.CRITICAL]
+            return f"❌ Validation failed with {len(critical)} critical issues"

data_contract_validator/extractors/__init__.py

@@ -0,0 +1,14 @@
+# data_contract_validator/extractors/__init__.py
+"""
+Schema extractors for different frameworks.
+"""
+
+from .base import BaseExtractor
+from .dbt import DBTExtractor
+from .fastapi import FastAPIExtractor
+
+__all__ = [
+    "BaseExtractor",
+    "DBTExtractor",
+    "FastAPIExtractor",
+]

data_contract_validator/extractors/base.py

@@ -0,0 +1,45 @@
+"""
+Base extractor interface for schema extraction.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+from ..core.models import Schema
+
+
+class BaseExtractor(ABC):
+    """Base class for all schema extractors."""
+
+    @abstractmethod
+    def extract_schemas(self) -> Dict[str, Schema]:
+        """
+        Extract schemas from the source.
+
+        Returns:
+            Dict mapping table names to Schema objects
+        """
+        pass
+
+    def _python_to_sql_type(self, python_type: str) -> str:
+        """Convert Python type hints to SQL types."""
+        type_mappings = {
+            "str": "varchar",
+            "int": "integer",
+            "float": "float",
+            "bool": "boolean",
+            "datetime": "timestamp",
+            "date": "date",
+            "list": "json",
+            "dict": "json",
+        }
+
+        # Handle Optional types
+        if "optional" in python_type.lower():
+            inner_type = python_type.lower().replace("optional[", "").replace("]", "")
+            return self._python_to_sql_type(inner_type)
+
+        return type_mappings.get(python_type.lower(), "varchar")
+
+    def _normalize_column_name(self, name: str) -> str:
+        """Normalize column names for comparison."""
+        return name.lower().strip()

data_contract_validator/extractors/dbt.py

@@ -0,0 +1,213 @@
+# data_contract_validator/extractors/dbt.py
+"""
+DBT schema extractor - simplified version of your working code.
+"""
+
+import json
+import subprocess
+import re
+from pathlib import Path
+from typing import Dict, List, Optional, Any
+
+from .base import BaseExtractor
+from ..core.models import Schema
+
+
+class DBTExtractor(BaseExtractor):
+    """Extract schemas from DBT projects."""
+
+    def __init__(self, project_path: str = "."):
+        self.project_path = Path(project_path)
+        self.target_dir = self.project_path / "target"
+        self.manifest_path = self.target_dir / "manifest.json"
+        self.models_path = self.project_path / "models"
+
+    def extract_schemas(self) -> Dict[str, Schema]:
+        """Extract schemas from DBT project."""
+        print(f"🔍 Extracting DBT schemas from {self.project_path}")
+
+        # Try manifest first, fallback to SQL parsing
+        if self._try_compile_dbt() and self.manifest_path.exists():
+            print("   📋 Using manifest.json")
+            return self._extract_from_manifest()
+        else:
+            print("   📄 Using SQL file parsing")
+            return self._extract_from_sql_files()
+
+    def _try_compile_dbt(self) -> bool:
+        """Try to compile DBT project."""
+        try:
+            result = subprocess.run(
+                ["dbt", "parse", "--project-dir", str(self.project_path)],
+                capture_output=True,
+                text=True,
+                timeout=60,
+            )
+            return result.returncode == 0
+        except:
+            return False
+
+    def _extract_from_manifest(self) -> Dict[str, Schema]:
+        """Extract schemas from manifest.json."""
+        with open(self.manifest_path, "r") as f:
+            manifest = json.load(f)
+
+        schemas = {}
+        for node_id, node in manifest.get("nodes", {}).items():
+            if node.get("resource_type") == "model":
+                model_name = node.get("alias") or node.get("name")
+
+                columns = []
+                for col_name, col_info in node.get("columns", {}).items():
+                    columns.append(
+                        {
+                            "name": col_name,
+                            "type": col_info.get("data_type", "varchar"),
+                            "required": True,
+                            "nullable": False,
+                        }
+                    )
+
+                schemas[model_name] = Schema(
+                    name=model_name, columns=columns, source="dbt_manifest"
+                )
+
+        print(f"   ✅ Found {len(schemas)} tables in manifest")
+        return schemas
+
+    def _extract_from_sql_files(self) -> Dict[str, Schema]:
+        """Extract schemas from SQL files directly."""
+        schemas = {}
+        sql_files = list(self.models_path.rglob("*.sql"))
+
+        print(f"   🔍 Found {len(sql_files)} SQL files to analyze")
+
+        for sql_file in sql_files:
+            model_name = sql_file.stem
+
+            # Skip test/analysis files
+            if any(skip in str(sql_file) for skip in ["tests", "analysis", "macros"]):
+                continue
+
+            try:
+                with open(sql_file, "r", encoding="utf-8") as f:
+                    sql_content = f.read()
+
+                columns = self._extract_columns_from_sql(sql_content)
+                if columns:
+                    schemas[model_name] = Schema(
+                        name=model_name, columns=columns, source="sql_parsing"
+                    )
+                    print(f"   📋 {model_name}: {len(columns)} columns")
+
+            except Exception as e:
+                print(f"   ❌ Error parsing {model_name}: {e}")
+
+        return schemas
+
+    def _extract_columns_from_sql(self, sql_content: str) -> List[Dict[str, Any]]:
+        """Extract columns from SQL content - simplified version."""
+        # Remove comments and Jinja
+        cleaned = re.sub(r"--.*?\n", "\n", sql_content)
+        cleaned = re.sub(r"/\*.*?\*/", "", cleaned, flags=re.DOTALL)
+        cleaned = re.sub(r"\{\{.*?\}\}", "", cleaned)
+
+        # Find final SELECT statement
+        select_matches = list(
+            re.finditer(r"select\s+(.*?)\s+from", cleaned, re.DOTALL | re.IGNORECASE)
+        )
+
+        if not select_matches:
+            return []
+
+        # Use the last SELECT (after CTEs)
+        select_content = select_matches[-1].group(1).strip()
+
+        # Split by comma and parse each column
+        columns = []
+        column_parts = self._split_columns(select_content)
+
+        for col_text in column_parts:
+            col_text = col_text.strip()
+            if col_text and col_text != "*":
+                column_name = self._extract_column_name(col_text)
+                if column_name:
+                    columns.append(
+                        {
+                            "name": column_name,
+                            "type": self._infer_data_type(col_text),
+                            "required": True,
+                            "nullable": False,
+                        }
+                    )
+
+        return columns
+
+    def _split_columns(self, select_clause: str) -> List[str]:
+        """Split SELECT columns by comma, handling nested functions."""
+        columns = []
+        current_column = ""
+        paren_depth = 0
+
+        for char in select_clause:
+            if char == "(":
+                paren_depth += 1
+            elif char == ")":
+                paren_depth -= 1
+            elif char == "," and paren_depth == 0:
+                if current_column.strip():
+                    columns.append(current_column.strip())
+                current_column = ""
+                continue
+
+            current_column += char
+
+        if current_column.strip():
+            columns.append(current_column.strip())
+
+        return columns
+
+    def _extract_column_name(self, col_text: str) -> Optional[str]:
+        """Extract clean column name from column definition."""
+        col_text = col_text.strip()
+
+        # Check for AS alias
+        as_match = re.search(r"\s+as\s+(\w+)$", col_text, re.IGNORECASE)
+        if as_match:
+            return as_match.group(1).lower()
+
+        # Handle table.column format
+        table_match = re.search(r"(\w+)\.(\w+)$", col_text)
+        if table_match:
+            return table_match.group(2).lower()
+
+        # Simple column name
+        simple_match = re.search(r"^(\w+)$", col_text)
+        if simple_match:
+            return simple_match.group(1).lower()
+
+        # For complex expressions, try to extract alias
+        parts = col_text.split()
+        if len(parts) > 1 and not "(" in parts[-1]:
+            return parts[-1].lower()
+
+        return None
+
+    def _infer_data_type(self, expression: str) -> str:
+        """Infer data type from SQL expression."""
+        expr_upper = expression.upper()
+
+        if any(func in expr_upper for func in ["COUNT", "SUM", "ROW_NUMBER"]):
+            return "integer"
+        elif "AVG" in expr_upper:
+            return "float"
+        elif any(func in expr_upper for func in ["CONCAT", "UPPER", "LOWER"]):
+            return "varchar"
+        elif "TIMESTAMP" in expr_upper or "CURRENT_TIMESTAMP" in expr_upper:
+            return "timestamp"
+        elif "DATE" in expr_upper:
+            return "date"
+        elif any(keyword in expr_upper for keyword in ["TRUE", "FALSE", "BOOLEAN"]):
+            return "boolean"
+        else:
+            return "varchar"