pycharter 0.0.20__py3-none-any.whl → 0.0.22__py3-none-any.whl

api/utils.py

@@ -0,0 +1,224 @@
+"""
+Shared utilities for API routes.
+
+This module provides common helper functions used across API endpoints,
+including UUID handling, database queries, and error handling.
+"""
+
+import logging
+import uuid
+from typing import Optional, TypeVar, Union
+
+from fastapi import HTTPException, status
+from sqlalchemy import cast, String
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.decl_api import DeclarativeMeta
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+def safe_uuid_to_str(value: Optional[Union[uuid.UUID, str]]) -> Optional[str]:
+    """
+    Safely convert a UUID or string to a string representation.
+    
+    Handles both UUID objects and string representations, including
+    cases where SQLite might store UUIDs as strings.
+    
+    Args:
+        value: UUID object, string, or None
+        
+    Returns:
+        String representation of UUID, or None if value is None
+    """
+    if value is None:
+        return None
+    if isinstance(value, uuid.UUID):
+        return str(value)
+    if isinstance(value, str):
+        # If it's already a string, validate it's a valid UUID format
+        try:
+            # Validate and normalize the UUID string
+            uuid_obj = uuid.UUID(value)
+            return str(uuid_obj)
+        except (ValueError, AttributeError):
+            # If it's not a valid UUID, return as-is (might be a schema name)
+            return value
+    # Fallback: convert to string
+    return str(value)
+
+
+def ensure_uuid(value: Optional[Union[uuid.UUID, str, bytes]]) -> Optional[uuid.UUID]:
+    """
+    Ensure a value is a UUID object, converting from string or bytes if needed.
+    
+    Args:
+        value: UUID object, string, bytes, or None
+        
+    Returns:
+        UUID object, or None if value is None
+        
+    Raises:
+        ValueError: If value cannot be converted to UUID
+    """
+    if value is None:
+        return None
+    if isinstance(value, uuid.UUID):
+        return value
+    if isinstance(value, str):
+        # Handle empty strings
+        if not value.strip():
+            return None
+        return uuid.UUID(value)
+    if isinstance(value, bytes):
+        # Handle bytes (sometimes database drivers return UUIDs as bytes)
+        return uuid.UUID(value.decode('utf-8'))
+    # Try to convert to string first, then to UUID
+    try:
+        return uuid.UUID(str(value))
+    except (ValueError, AttributeError) as e:
+        raise ValueError(f"Cannot convert {type(value)} ({value}) to UUID: {e}")
+
+
+def get_by_id_with_fallback(
+    db: Session,
+    model: DeclarativeMeta,
+    id_value: Union[uuid.UUID, str],
+    error_message: Optional[str] = None,
+) -> Optional[T]:
+    """
+    Get a database record by ID with fallback strategies for UUID compatibility.
+    
+    This function tries multiple strategies to find a record:
+    1. Direct UUID comparison
+    2. String comparison (for SQLite compatibility)
+    3. Query all and compare by string (final fallback)
+    
+    Args:
+        db: Database session
+        model: SQLAlchemy model class
+        id_value: UUID or string ID to search for
+        error_message: Optional custom error message if not found
+        
+    Returns:
+        Model instance if found, None otherwise
+    """
+    # Try UUID comparison first
+    try:
+        if isinstance(id_value, str):
+            id_uuid = uuid.UUID(id_value)
+        else:
+            id_uuid = id_value
+        
+        record = db.query(model).filter(model.id == id_uuid).first()
+        if record:
+            return record
+    except (ValueError, TypeError):
+        pass
+    
+    # Try string comparison (for SQLite compatibility)
+    id_str = safe_uuid_to_str(id_value)
+    if id_str:
+        try:
+            record = db.query(model).filter(
+                cast(model.id, String) == id_str
+            ).first()
+            if record:
+                return record
+        except Exception:
+            pass
+    
+    # Final fallback: query all and compare by string
+    try:
+        all_records = db.query(model).all()
+        for record in all_records:
+            record_id_str = safe_uuid_to_str(record.id)
+            if record_id_str and id_str:
+                if (record_id_str == id_str or 
+                    record_id_str.lower() == id_str.lower()):
+                    return record
+    except Exception as e:
+        logger.debug(f"Fallback query failed: {e}")
+    
+    return None
+
+
+def get_by_id_or_404(
+    db: Session,
+    model: DeclarativeMeta,
+    id_value: Union[uuid.UUID, str],
+    error_message: Optional[str] = None,
+    model_name: Optional[str] = None,
+) -> T:
+    """
+    Get a database record by ID or raise 404 if not found.
+    
+    Args:
+        db: Database session
+        model: SQLAlchemy model class
+        id_value: UUID or string ID to search for
+        error_message: Optional custom error message
+        model_name: Optional model name for error message
+        
+    Returns:
+        Model instance
+        
+    Raises:
+        HTTPException: 404 if record not found
+    """
+    record = get_by_id_with_fallback(db, model, id_value, error_message)
+    
+    if not record:
+        id_str = safe_uuid_to_str(id_value)
+        model_display = model_name or model.__name__
+        default_message = f"{model_display} with ID {id_str} not found"
+        message = error_message or default_message
+        
+        # Log diagnostic info
+        total = db.query(model).count()
+        logger.error(
+            f"{model_display} not found: {id_str}. "
+            f"Total {model_display.lower()}s: {total}"
+        )
+        
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=message,
+        )
+    
+    return record
+
+
+def model_to_dict(model_instance) -> dict:
+    """
+    Convert SQLAlchemy model instance to dictionary.
+    
+    Handles UUIDs, datetimes, and JSON fields appropriately.
+    
+    Args:
+        model_instance: SQLAlchemy model instance
+        
+    Returns:
+        Dictionary representation of the model
+    """
+    if model_instance is None:
+        return {}
+    
+    result = {}
+    for column in model_instance.__table__.columns:
+        value = getattr(model_instance, column.name, None)
+        if value is None:
+            result[column.name] = None
+        # Convert UUID to string
+        elif hasattr(value, 'hex'):
+            result[column.name] = str(value)
+        # Convert datetime to ISO format string
+        elif hasattr(value, 'isoformat'):
+            result[column.name] = value.isoformat()
+        # Keep JSON fields as-is
+        elif isinstance(value, (dict, list)):
+            result[column.name] = value
+        else:
+            result[column.name] = value
+    return result

pycharter/__init__.py

@@ -1,35 +1,109 @@
 """
 PyCharter - Data Contract Management and Validation
 
-Six core services:
+Core Services:
 1. Contract Parser - Reads and decomposes data contract files
-1b. Contract Builder - Constructs consolidated contracts from separate artifacts
-2. Metadata Store Client - Database operations for metadata storage
-3. Pydantic Generator - Generates Pydantic models from JSON Schema
-4. JSON Schema Converter - Converts Pydantic models to JSON Schema
-5. Runtime Validator - Lightweight validation utility
+2. Contract Builder - Constructs consolidated contracts from separate artifacts
+3. Metadata Store - Database operations for metadata storage
+4. Pydantic Generator - Generates Pydantic models from JSON Schema
+5. JSON Schema Converter - Converts Pydantic models to JSON Schema
+6. Runtime Validator - Validation utilities
+7. Quality Assurance - Data quality checking and monitoring
+
+API Organization:
+- Tier 1: Primary Interfaces (Classes - Use these for best performance)
+- Tier 2: Convenience Functions (Quick start - one-off use cases)
+- Tier 3: Low-Level Utilities (When you already have models/schemas)
 """
 
-__version__ = "0.0.2"
+__version__ = "0.0.22"
 
-# Service 1b: Contract Builder
-from pycharter.contract_builder import (
-    ContractArtifacts,
-    build_contract,
-    build_contract_from_store,
+# ============================================================================
+# TIER 1: PRIMARY INTERFACES (Classes - Use these for best performance)
+# ============================================================================
+
+# Runtime Validator (PRIMARY INTERFACE)
+from pycharter.runtime_validator import (
+    Validator,  # ⭐ PRIMARY: Use this for validation
+    create_validator,
+)
+
+# Quality Assurance (PRIMARY INTERFACE)
+from pycharter.quality import (
+    QualityCheck,  # ⭐ PRIMARY: Use this for quality checks
+    QualityCheckOptions,
+    QualityReport,
+    QualityThresholds,
 )
 
-# Service 1: Contract Parser
+# Metadata Store (PRIMARY INTERFACE)
+from pycharter.metadata_store import MetadataStoreClient
+
+# ============================================================================
+# TIER 2: CONVENIENCE FUNCTIONS (Quick start - one-off use cases)
+# ============================================================================
+
+# Contract Parser
 from pycharter.contract_parser import (
     ContractMetadata,
     parse_contract,
     parse_contract_file,
 )
 
-# Service 2: Metadata Store Client
-from pycharter.metadata_store import MetadataStoreClient
+# Contract Builder
+from pycharter.contract_builder import (
+    ContractArtifacts,
+    build_contract,
+    build_contract_from_store,
+)
+
+# Pydantic Generator - Input type helpers (convenience)
+from pycharter.pydantic_generator import (
+    from_dict,  # Quick: schema from dict
+    from_file,  # Quick: schema from file
+    from_json,  # Quick: schema from JSON string
+    from_url,  # Quick: schema from URL
+    generate_model,  # Advanced: when you need more control
+    generate_model_file,
+)
+
+# JSON Schema Converter - Output type helpers (convenience)
+from pycharter.json_schema_converter import (
+    to_dict,  # Quick: schema to dict
+    to_file,  # Quick: schema to file
+    to_json,  # Quick: schema to JSON string
+    model_to_schema,  # Advanced: core conversion
+)
+
+# Runtime Validator - Data source helpers (convenience)
+from pycharter.runtime_validator import (
+    ValidationResult,
+    # Quick validation functions (use Validator class for multiple validations)
+    validate_with_store,  # Quick: validate with metadata store
+    validate_batch_with_store,  # Quick: batch validate with metadata store
+    validate_with_contract,  # Quick: validate with contract file/dict
+    validate_batch_with_contract,  # Quick: batch validate with contract file/dict
+    get_model_from_store,  # Quick: get model from store
+    get_model_from_contract,  # Quick: get model from contract
+    # Decorators
+    validate_input,
+    validate_output,
+    validate_with_contract_decorator,
+)
+
+# ============================================================================
+# TIER 3: LOW-LEVEL UTILITIES (When you already have models/schemas)
+# ============================================================================
+
+from pycharter.runtime_validator import (
+    validate,  # Low-level: validate with existing model
+    validate_batch,  # Low-level: batch validate with existing model
+)
+
+# ============================================================================
+# METADATA STORE IMPLEMENTATIONS
+# ============================================================================
 
-# Optional metadata store implementations
 try:
     from pycharter.metadata_store import InMemoryMetadataStore
 except ImportError:
@@ -50,104 +124,86 @@ try:
 except ImportError:
     RedisMetadataStore = None  # type: ignore[assignment,misc]
 
-# Service 4: JSON Schema Converter
-from pycharter.json_schema_converter import (
-    model_to_schema,
-    to_dict,
-    to_file,
-    to_json,
-)
-
-# Service 3: Pydantic Generator
-from pycharter.pydantic_generator import (
-    from_dict,
-    from_file,
-    from_json,
-    from_url,
-    generate_model,
-    generate_model_file,
-)
+try:
+    from pycharter.metadata_store import SQLiteMetadataStore
+except ImportError:
+    SQLiteMetadataStore = None  # type: ignore[assignment,misc]
 
-# Service 5: Runtime Validator
-from pycharter.runtime_validator import (
-    ValidationResult,
-    get_model_from_contract,
-    get_model_from_store,
-    validate,
-    validate_batch,
-    validate_batch_with_contract,
-    validate_batch_with_store,
-    validate_input,
-    validate_output,
-    validate_with_contract,
-    validate_with_contract_decorator,
-    validate_with_store,
-)
+# ============================================================================
+# QUALITY ASSURANCE - Additional utilities (if needed for advanced use)
+# ============================================================================
 
-# Service 6: Quality Assurance
 from pycharter.quality import (
-    DataProfiler,
-    FieldQualityMetrics,
-    QualityCheck,
-    QualityCheckOptions,
     QualityMetrics,
-    QualityReport,
     QualityScore,
-    QualityThresholds,
-    ViolationRecord,
+    FieldQualityMetrics,
     ViolationTracker,
+    ViolationRecord,
+    DataProfiler,
 )
 
 __all__ = [
-    # Contract Parser
+    # ========================================================================
+    # TIER 1: PRIMARY INTERFACES
+    # ========================================================================
+    "Validator",  # ⭐ PRIMARY: Use for validation
+    "create_validator",
+    "QualityCheck",  # ⭐ PRIMARY: Use for quality checks
+    "QualityCheckOptions",
+    "QualityReport",
+    "QualityThresholds",
+    "MetadataStoreClient",
+    # ========================================================================
+    # TIER 2: CONVENIENCE FUNCTIONS
+    # ========================================================================
+    # Contract Management
     "parse_contract",
     "parse_contract_file",
     "ContractMetadata",
-    # Contract Builder
     "build_contract",
     "build_contract_from_store",
     "ContractArtifacts",
-    # Metadata Store Client
-    "MetadataStoreClient",
+    # Pydantic Generator (input type helpers)
+    "from_dict",  # Quick: dict → model
+    "from_file",  # Quick: file → model
+    "from_json",  # Quick: JSON string → model
+    "from_url",  # Quick: URL → model
+    "generate_model",  # Advanced: more control
+    "generate_model_file",
+    # JSON Schema Converter (output type helpers)
+    "to_dict",  # Quick: model → dict
+    "to_file",  # Quick: model → file
+    "to_json",  # Quick: model → JSON string
+    "model_to_schema",  # Advanced: core conversion
+    # Runtime Validator (data source helpers)
+    "ValidationResult",
+    "validate_with_store",  # Quick: store → validate
+    "validate_batch_with_store",  # Quick: batch validate with store
+    "validate_with_contract",  # Quick: contract → validate
+    "validate_batch_with_contract",  # Quick: batch validate with contract
+    "get_model_from_store",  # Quick: store → model
+    "get_model_from_contract",  # Quick: contract → model
+    "validate_input",  # Decorator
+    "validate_output",  # Decorator
+    "validate_with_contract_decorator",
+    # ========================================================================
+    # TIER 3: LOW-LEVEL UTILITIES
+    # ========================================================================
+    "validate",  # Low-level: model → validate
+    "validate_batch",  # Low-level: model → batch validate
+    # ========================================================================
+    # METADATA STORE IMPLEMENTATIONS
+    # ========================================================================
     "InMemoryMetadataStore",
     "MongoDBMetadataStore",
     "PostgresMetadataStore",
     "RedisMetadataStore",
-    # Pydantic Generator
-    "generate_model",
-    "generate_model_file",
-    "from_dict",
-    "from_file",
-    "from_json",
-    "from_url",
-    # JSON Schema Converter
-    "to_dict",
-    "to_file",
-    "to_json",
-    "model_to_schema",
-    # Runtime Validator
-    "validate",
-    "validate_batch",
-    "ValidationResult",
-    # Database-backed validation
-    "validate_with_store",
-    "validate_batch_with_store",
-    "get_model_from_store",
-    # Contract-based validation (no database)
-    "validate_with_contract",
-    "validate_batch_with_contract",
-    "get_model_from_contract",
-    # Decorators
-    "validate_input",
-    "validate_output",
-    "validate_with_contract_decorator",
-    # Quality Assurance
-    "QualityCheck",
+    "SQLiteMetadataStore",
+    # ========================================================================
+    # QUALITY ASSURANCE - Additional utilities
+    # ========================================================================
     "QualityMetrics",
     "QualityScore",
-    "QualityReport",
-    "QualityThresholds",
-    "QualityCheckOptions",
     "FieldQualityMetrics",
     "ViolationTracker",
     "ViolationRecord",

pycharter/data/templates/template_transform_advanced.yaml

@@ -0,0 +1,50 @@
+# Advanced Transformation Template
+# Use this template for complex transformations with JSONata and custom functions
+
+title: advanced_transformation
+description: Advanced transformation with JSONata and custom functions
+version: 1.0.0
+
+# Step 1: Simple operations (optional but recommended for basic cleanup)
+transform:
+  rename:
+    oldName: new_name
+  convert:
+    price: float
+    timestamp: datetime
+  defaults:
+    source: "api"
+
+# Step 2: JSONata for complex transformations
+jsonata:
+  expression: |
+    $.{
+      "ticker": symbol,
+      "price_change": price - previousClose,
+      "price_change_pct": ((price - previousClose) / previousClose) * 100,
+      "is_positive": price > previousClose,
+      "formatted_date": $fromMillis(timestamp),
+      "metadata": {
+        "source": source,
+        "processed_at": $now()
+      }
+    }
+  mode: "record"  # Apply to each record individually
+  # mode: "batch"  # Apply to entire dataset
+
+# Step 3: Custom Python function for business logic
+custom_function:
+  # Option 1: Using module and function
+  module: "myproject.transforms"
+  function: "optimize_portfolio"
+  mode: "batch"
+  kwargs:
+    method: "min_volatility"
+    solver: "ipopt"
+    constraints:
+      max_weight: 0.1
+      min_weight: 0.01
+
+  # Option 2: Using callable path (alternative)
+  # callable: "myproject.transforms.optimize_portfolio"
+  # mode: "batch"

pycharter/data/templates/template_transform_simple.yaml

@@ -0,0 +1,59 @@
+# Simple Transformation Template
+# Use this template for basic field transformations
+
+title: simple_transformation
+description: Simple transformation configuration
+version: 1.0.0
+
+# Simple operations (applied in order: rename → convert → defaults → add → select → drop)
+transform:
+  # Rename fields (old_name: new_name)
+  rename:
+    oldName: new_name
+    camelCase: snake_case
+    userId: user_id
+  
+  # Convert field types
+  convert:
+    price: float
+    quantity: integer
+    active: boolean
+    created_at: datetime
+  
+  # Set default values for missing fields
+  defaults:
+    status: "pending"
+    priority: 0
+  
+  # Add computed fields
+  add:
+    full_name: "${first_name} ${last_name}"
+    created_at: "now()"
+    record_id: "uuid()"
+  
+  # Keep only these fields (removes all others)
+  # select:
+  #   - field1
+  #   - field2
+  
+  # Remove these fields (keeps all others)
+  # drop:
+  #   - internal_id
+  #   - debug_info
+
+# Advanced: JSONata for complex transformations (optional)
+# jsonata:
+#   expression: |
+#     $.{
+#       "ticker": symbol,
+#       "avg_price": $average(prices)
+#     }
+#   mode: "record"  # or "batch"
+
+# Advanced: Custom Python function (optional)
+# custom_function:
+#   module: "myproject.transforms"
+#   function: "optimize_data"
+#   mode: "batch"
+#   kwargs:
+#     method: "min_volatility"

pycharter/db/models/base.py

@@ -3,8 +3,7 @@ Base SQLAlchemy configuration
 """
 
 from sqlalchemy import create_engine, event
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import declarative_base, sessionmaker
 from sqlalchemy.schema import CreateTable
 
 Base = declarative_base()