additory 0.1.0a2__py3-none-any.whl → 0.1.0a4__py3-none-any.whl

additory/synthetic/schema_parser.py

@@ -1,685 +0,0 @@
-"""
-Schema parser for TOML schema processing.
-
-Handles parsing of TOML schema files with import declarations, inline pattern overrides,
-distribution strategy specifications, and comprehensive validation.
-
-Enhanced to support:
-- New [generation] section with imports and prefer_mode
-- Pattern type detection (inline_list, inline_regex, reference)
-- Integration with common module for pattern resolution
-"""
-
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Dict, List, Optional, Any, Union, Set
-import toml
-import logging
-
-from .models import (
-    SchemaDefinition, FieldDefinition, DistributionStrategy, DistributionType,
-    ValidationResult, ValidationStatus
-)
-from .file_managers import SchemaFileManager, ParsedSchemaFile
-from .pattern_resolver import PatternHierarchyResolver
-from .common_integration import SyntheticPatternLoader, detect_pattern_type_from_toml
-from .exceptions import (
-    SchemaParsingError, ValidationError, PatternResolutionError,
-    DistributionValidationError
-)
-
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class ParsedFieldDefinition:
-    """Represents a parsed field definition from schema."""
-    name: str
-    pattern_value: Union[str, List[str]]  # Can be reference, regex, or list
-    pattern_type: str  # "inline_list", "inline_regex", or "reference"
-    distribution: Optional[DistributionStrategy] = None
-    constraints: List[str] = field(default_factory=list)
-    metadata: Dict[str, Any] = field(default_factory=dict)
-
-
-@dataclass
-class SchemaParsingResult:
-    """Result of comprehensive schema parsing."""
-    schema_definition: SchemaDefinition
-    validation_result: ValidationResult
-    referenced_patterns: Set[str]
-    distribution_strategies: Dict[str, DistributionStrategy]
-    parsing_metadata: Dict[str, Any] = field(default_factory=dict)
-
-
-class SchemaParser:
-    """
-    Comprehensive TOML schema parser with validation and pattern resolution.
-    
-    Handles:
-    - Import declaration parsing and validation
-    - Inline pattern override processing
-    - Distribution strategy specification parsing
-    - Pattern reference validation
-    - Schema structure validation
-    """
-    
-    def __init__(self, pattern_resolver: Optional[PatternHierarchyResolver] = None,
-                 use_common_integration: bool = True):
-        """
-        Initialize the schema parser.
-        
-        Args:
-            pattern_resolver: Pattern hierarchy resolver for validation (legacy)
-            use_common_integration: Whether to use new common_integration module (default: True)
-        """
-        self.schema_file_manager = SchemaFileManager()
-        self.pattern_resolver = pattern_resolver or PatternHierarchyResolver()
-        
-        # New: Use common_integration for pattern loading
-        self.use_common_integration = use_common_integration
-        if use_common_integration:
-            self.pattern_loader = SyntheticPatternLoader()
-        
-        # Supported distribution types for validation
-        self.supported_distributions = {
-            'equal', 'custom', 'categorical', 'high_cardinality', 
-            'numeric_range', 'skewed'
-        }
-    
-    def parse_schema_file(self, file_path: str, validate_patterns: bool = True) -> SchemaParsingResult:
-        """
-        Parse a TOML schema file with comprehensive validation.
-        
-        Args:
-            file_path: Path to the TOML schema file
-            validate_patterns: Whether to validate referenced patterns exist
-            
-        Returns:
-            SchemaParsingResult with parsed schema and validation results
-            
-        Raises:
-            SchemaParsingError: If schema parsing fails
-            ValidationError: If validation fails
-        """
-        # Load the raw TOML file
-        parsed_file = self.schema_file_manager.load_toml_schema(file_path)
-        
-        # Initialize validation result
-        validation_result = ValidationResult(is_valid=True)
-        
-        # Parse field definitions from schema section
-        field_definitions, field_validation = self._parse_field_definitions(
-            parsed_file.schema_definitions, file_path
-        )
-        validation_result.merge(field_validation)
-        
-        # Parse distribution strategies
-        distribution_strategies, dist_validation = self._parse_distribution_strategies(
-            parsed_file.schema_definitions, file_path
-        )
-        validation_result.merge(dist_validation)
-        
-        # Validate imports
-        import_validation = self._validate_imports(parsed_file.imports, file_path)
-        validation_result.merge(import_validation)
-        
-        # Validate inline patterns
-        inline_validation = self._validate_inline_patterns(parsed_file.inline_patterns, file_path)
-        validation_result.merge(inline_validation)
-        
-        # Collect all referenced patterns
-        referenced_patterns = self._collect_referenced_patterns(
-            field_definitions, parsed_file.inline_patterns
-        )
-        
-        # Validate pattern references if requested
-        if validate_patterns:
-            pattern_validation = self._validate_pattern_references(
-                referenced_patterns, parsed_file.imports, 
-                parsed_file.inline_patterns, file_path
-            )
-            validation_result.merge(pattern_validation)
-        
-        # Create schema definition
-        schema_definition = SchemaDefinition(
-            imports=parsed_file.imports,
-            inline_patterns=parsed_file.inline_patterns,
-            field_definitions={name: self._convert_to_field_definition(field_def, distribution_strategies.get(name))
-                             for name, field_def in field_definitions.items()},
-            metadata=parsed_file.metadata,
-            source_file=file_path
-        )
-        
-        # Create parsing metadata
-        parsing_metadata = {
-            'file_path': file_path,
-            'field_count': len(field_definitions),
-            'import_count': len(parsed_file.imports),
-            'inline_pattern_count': len(parsed_file.inline_patterns),
-            'distribution_strategy_count': len(distribution_strategies),
-            'referenced_pattern_count': len(referenced_patterns)
-        }
-        
-        return SchemaParsingResult(
-            schema_definition=schema_definition,
-            validation_result=validation_result,
-            referenced_patterns=referenced_patterns,
-            distribution_strategies=distribution_strategies,
-            parsing_metadata=parsing_metadata
-        )
-    
-    def _parse_field_definitions(self, schema_definitions: Dict[str, Any], 
-                                file_path: str) -> tuple[Dict[str, ParsedFieldDefinition], ValidationResult]:
-        """
-        Parse field definitions from schema section.
-        
-        Enhanced to support:
-        - String patterns (reference or inline regex)
-        - Array patterns (inline list)
-        - Pattern type detection
-        
-        Args:
-            schema_definitions: Schema definitions from TOML
-            file_path: File path for error reporting
-            
-        Returns:
-            Tuple of (field_definitions, validation_result)
-        """
-        field_definitions = {}
-        validation_result = ValidationResult(is_valid=True)
-        
-        for field_name, pattern_spec in schema_definitions.items():
-            try:
-                # Detect pattern type
-                pattern_type = detect_pattern_type_from_toml(pattern_spec)
-                
-                if pattern_type == "unknown":
-                    validation_result.add_error(
-                        f"Invalid pattern type for field '{field_name}'",
-                        f"Pattern must be a string (reference/regex) or array (list), got {type(pattern_spec)}"
-                    )
-                    continue
-                
-                # Validate field name
-                if not self._is_valid_field_name(field_name):
-                    validation_result.add_error(
-                        f"Invalid field name '{field_name}' in schema",
-                        "Field names must be valid identifiers"
-                    )
-                    continue
-                
-                # Create field definition with pattern value and type
-                field_def = ParsedFieldDefinition(
-                    name=field_name,
-                    pattern_value=pattern_spec,
-                    pattern_type=pattern_type
-                )
-                
-                field_definitions[field_name] = field_def
-                
-                logger.debug(f"Parsed field '{field_name}': type={pattern_type}, value={pattern_spec}")
-                
-            except Exception as e:
-                validation_result.add_error(
-                    f"Error parsing field '{field_name}': {e}",
-                    "Check field definition syntax"
-                )
-        
-        return field_definitions, validation_result
-    
-    def _parse_distribution_strategies(self, schema_definitions: Dict[str, Any], 
-                                     file_path: str) -> tuple[Dict[str, DistributionStrategy], ValidationResult]:
-        """
-        Parse distribution strategy specifications from schema.
-        
-        Enhanced to handle both string and array patterns.
-        
-        Args:
-            schema_definitions: Schema definitions from TOML
-            file_path: File path for error reporting
-            
-        Returns:
-            Tuple of (distribution_strategies, validation_result)
-        """
-        distribution_strategies = {}
-        validation_result = ValidationResult(is_valid=True)
-        
-        # Look for distribution specifications in metadata or special sections
-        # For now, we'll parse simple distribution specs from pattern specifications
-        for field_name, pattern_spec in schema_definitions.items():
-            try:
-                # Skip non-string patterns (arrays don't have distribution specs in the pattern)
-                if not isinstance(pattern_spec, str):
-                    continue
-                
-                distribution = self._extract_distribution_from_spec(pattern_spec)
-                if distribution:
-                    # Validate distribution strategy
-                    dist_validation = self._validate_distribution_strategy(distribution, field_name)
-                    validation_result.merge(dist_validation)
-                    
-                    if dist_validation.is_valid:
-                        distribution_strategies[field_name] = distribution
-                        
-            except Exception as e:
-                validation_result.add_error(
-                    f"Error parsing distribution for field '{field_name}': {e}",
-                    "Check distribution strategy syntax"
-                )
-        
-        return distribution_strategies, validation_result
-    
-    def _validate_imports(self, imports: List[str], file_path: str) -> ValidationResult:
-        """
-        Validate import declarations.
-        
-        Enhanced to use common_integration for validation.
-        
-        Args:
-            imports: List of import declarations
-            file_path: File path for error reporting
-            
-        Returns:
-            ValidationResult with import validation status
-        """
-        validation_result = ValidationResult(is_valid=True)
-        
-        # Use common_integration for validation if enabled
-        if self.use_common_integration and hasattr(self, 'pattern_loader'):
-            is_valid, errors = self.pattern_loader.validate_imports(imports)
-            if not is_valid:
-                for error in errors:
-                    validation_result.add_error(error, "Check import declarations")
-            return validation_result
-        
-        # Legacy validation
-        for import_name in imports:
-            # Validate import name format
-            if not self._is_valid_import_name(import_name):
-                validation_result.add_error(
-                    f"Invalid import name '{import_name}'",
-                    "Import names should be valid pattern file names (without .properties extension)"
-                )
-            
-            # Check for circular imports (basic check)
-            if import_name == "self" or import_name == Path(file_path).stem:
-                validation_result.add_error(
-                    f"Circular import detected: '{import_name}'",
-                    "Schemas cannot import themselves"
-                )
-        
-        return validation_result
-    
-    def _validate_inline_patterns(self, inline_patterns: Dict[str, Union[str, List[str]]], 
-                                 file_path: str) -> ValidationResult:
-        """
-        Validate inline pattern definitions.
-        
-        Enhanced to support both string (regex) and array (list) patterns.
-        
-        Args:
-            inline_patterns: Dictionary of inline patterns
-            file_path: File path for error reporting
-            
-        Returns:
-            ValidationResult with inline pattern validation status
-        """
-        validation_result = ValidationResult(is_valid=True)
-        
-        for pattern_name, pattern_value in inline_patterns.items():
-            # Validate pattern name
-            if not self._is_valid_pattern_name(pattern_name):
-                validation_result.add_error(
-                    f"Invalid inline pattern name '{pattern_name}'",
-                    "Pattern names must start with a letter and contain only letters, numbers, and underscores"
-                )
-            
-            # Validate based on pattern type
-            if isinstance(pattern_value, str):
-                # String pattern (regex)
-                if not pattern_value.strip():
-                    validation_result.add_error(
-                        f"Empty regex pattern for inline pattern '{pattern_name}'",
-                        "Regex patterns must not be empty"
-                    )
-                
-                # Basic regex syntax validation
-                try:
-                    import re
-                    re.compile(pattern_value)
-                except re.error as e:
-                    validation_result.add_error(
-                        f"Invalid regex syntax in inline pattern '{pattern_name}': {e}",
-                        "Check regex syntax for proper escaping and structure"
-                    )
-            
-            elif isinstance(pattern_value, list):
-                # Array pattern (list)
-                if not pattern_value:
-                    validation_result.add_error(
-                        f"Empty list for inline pattern '{pattern_name}'",
-                        "List patterns must contain at least one value"
-                    )
-                
-                # Validate all items are strings
-                for i, item in enumerate(pattern_value):
-                    if not isinstance(item, str):
-                        validation_result.add_error(
-                            f"Invalid item type in inline pattern '{pattern_name}' at index {i}",
-                            f"All list items must be strings, got {type(item)}"
-                        )
-            
-            else:
-                validation_result.add_error(
-                    f"Invalid pattern type for inline pattern '{pattern_name}'",
-                    f"Pattern must be a string (regex) or array (list), got {type(pattern_value)}"
-                )
-        
-        return validation_result
-    
-    def _collect_referenced_patterns(self, field_definitions: Dict[str, ParsedFieldDefinition],
-                                   inline_patterns: Dict[str, Union[str, List[str]]]) -> Set[str]:
-        """
-        Collect all pattern names referenced in the schema.
-        
-        Enhanced to only collect references (not inline patterns).
-        
-        Args:
-            field_definitions: Parsed field definitions
-            inline_patterns: Inline pattern definitions
-            
-        Returns:
-            Set of all referenced pattern names (excluding inline patterns)
-        """
-        referenced_patterns = set()
-        
-        # Add patterns from field definitions (only references, not inline)
-        for field_def in field_definitions.values():
-            if field_def.pattern_type == "reference":
-                # Only references need to be resolved
-                referenced_patterns.add(field_def.pattern_value)
-        
-        # Inline patterns don't need resolution (they're already defined)
-        # But we track them for completeness
-        referenced_patterns.update(inline_patterns.keys())
-        
-        return referenced_patterns
-    
-    def _validate_pattern_references(self, referenced_patterns: Set[str],
-                                   imports: List[str], inline_patterns: Dict[str, Union[str, List[str]]],
-                                   file_path: str) -> ValidationResult:
-        """
-        Validate that all referenced patterns can be resolved.
-        
-        Enhanced to use common_integration for pattern resolution.
-        
-        Args:
-            referenced_patterns: Set of referenced pattern names
-            imports: List of import declarations
-            inline_patterns: Inline pattern definitions
-            file_path: File path for error reporting
-            
-        Returns:
-            ValidationResult with pattern reference validation status
-        """
-        validation_result = ValidationResult(is_valid=True)
-        
-        # Get prefer_mode from parsed file metadata
-        prefer_mode = "default"  # Default value
-        
-        # Use common_integration for validation if enabled
-        if self.use_common_integration and hasattr(self, 'pattern_loader'):
-            for pattern_name in referenced_patterns:
-                # Skip inline patterns (they're already defined)
-                if pattern_name in inline_patterns:
-                    continue
-                
-                try:
-                    # Try to resolve the pattern using common_integration
-                    self.pattern_loader.load_pattern(
-                        pattern_name,
-                        imports=imports,
-                        prefer_mode=prefer_mode
-                    )
-                    logger.debug(f"Successfully resolved pattern '{pattern_name}'")
-                except PatternResolutionError as e:
-                    validation_result.add_error(
-                        f"Cannot resolve pattern '{pattern_name}' referenced in schema",
-                        f"Pattern not found in imports {imports} or inline patterns. {str(e)}"
-                    )
-        else:
-            # Legacy validation using PatternHierarchyResolver
-            for pattern_name in referenced_patterns:
-                try:
-                    # Try to resolve the pattern
-                    self.pattern_resolver.resolve_pattern(
-                        pattern_name,
-                        inline_patterns=inline_patterns,
-                        user_imports=imports
-                    )
-                except PatternResolutionError as e:
-                    validation_result.add_error(
-                        f"Cannot resolve pattern '{pattern_name}' referenced in schema",
-                        f"Pattern not found in imports {imports} or inline patterns. {str(e)}"
-                    )
-        
-        return validation_result
-    
-    def _parse_pattern_specification(self, pattern_spec: str) -> tuple[str, List[str]]:
-        """
-        Parse a pattern specification that might include constraints.
-        
-        Args:
-            pattern_spec: Pattern specification string
-            
-        Returns:
-            Tuple of (pattern_name, constraints)
-        """
-        # For now, simple parsing - just return the pattern name
-        # Future enhancement: parse constraints like "email|min_length:5|max_length:50"
-        parts = pattern_spec.split('|')
-        pattern_name = parts[0].strip()
-        constraints = [part.strip() for part in parts[1:]] if len(parts) > 1 else []
-        
-        return pattern_name, constraints
-    
-    def _extract_distribution_from_spec(self, pattern_spec: str) -> Optional[DistributionStrategy]:
-        """
-        Extract distribution strategy from pattern specification.
-        
-        Args:
-            pattern_spec: Pattern specification string
-            
-        Returns:
-            DistributionStrategy if found, None otherwise
-        """
-        # Look for distribution specifications in the pattern spec
-        # Format: "pattern_name|distribution:equal" or "pattern_name|distribution:custom:20,30,50"
-        parts = pattern_spec.split('|')
-        
-        for part in parts:
-            part = part.strip()
-            if part.startswith('distribution:'):
-                dist_spec = part[13:]  # Remove 'distribution:' prefix
-                return self._parse_distribution_spec(dist_spec)
-        
-        return None
-    
-    def _parse_distribution_spec(self, dist_spec: str) -> DistributionStrategy:
-        """
-        Parse a distribution specification string.
-        
-        Args:
-            dist_spec: Distribution specification (e.g., "equal", "custom:20,30,50")
-            
-        Returns:
-            DistributionStrategy object
-            
-        Raises:
-            DistributionValidationError: If distribution spec is invalid
-        """
-        parts = dist_spec.split(':')
-        dist_type = parts[0].strip()
-        
-        if dist_type not in self.supported_distributions:
-            raise DistributionValidationError(
-                f"Unsupported distribution type '{dist_type}'",
-                dist_type,
-                f"Supported types: {', '.join(self.supported_distributions)}"
-            )
-        
-        # Parse distribution parameters
-        parameters = {}
-        if len(parts) > 1:
-            param_str = parts[1].strip()
-            if dist_type == 'custom':
-                # Parse weights: "20,30,50"
-                try:
-                    weights = [float(w.strip()) for w in param_str.split(',')]
-                    parameters['weights'] = weights
-                except ValueError:
-                    raise DistributionValidationError(
-                        f"Invalid weights specification '{param_str}'",
-                        dist_type,
-                        "Weights must be comma-separated numbers"
-                    )
-            elif dist_type == 'categorical':
-                # Parse categories: "A,B,C"
-                categories = [cat.strip() for cat in param_str.split(',')]
-                parameters['categories'] = categories
-            elif dist_type == 'numeric_range':
-                # Parse range: "1,100"
-                try:
-                    range_parts = param_str.split(',')
-                    if len(range_parts) == 2:
-                        parameters['min'] = float(range_parts[0].strip())
-                        parameters['max'] = float(range_parts[1].strip())
-                    else:
-                        raise ValueError("Range must have exactly 2 values")
-                except ValueError:
-                    raise DistributionValidationError(
-                        f"Invalid range specification '{param_str}'",
-                        dist_type,
-                        "Range must be in format 'min,max'"
-                    )
-        
-        return DistributionStrategy(
-            strategy_type=DistributionType(dist_type),
-            parameters=parameters
-        )
-    
-    def _validate_distribution_strategy(self, distribution: DistributionStrategy, 
-                                      field_name: str) -> ValidationResult:
-        """
-        Validate a distribution strategy.
-        
-        Args:
-            distribution: Distribution strategy to validate
-            field_name: Field name for error reporting
-            
-        Returns:
-            ValidationResult with distribution validation status
-        """
-        validation_result = ValidationResult(is_valid=True)
-        
-        try:
-            # Validate based on distribution type
-            if distribution.strategy_type == DistributionType.CUSTOM:
-                weights = distribution.parameters.get('weights', [])
-                if not weights:
-                    validation_result.add_error(
-                        f"Custom distribution for field '{field_name}' missing weights",
-                        "Custom distributions require weight parameters"
-                    )
-                elif abs(sum(weights) - 100.0) > 0.01:
-                    validation_result.add_error(
-                        f"Custom distribution weights for field '{field_name}' must sum to 100",
-                        f"Current sum: {sum(weights)}"
-                    )
-            
-            elif distribution.strategy_type == DistributionType.CATEGORICAL:
-                categories = distribution.parameters.get('categories', [])
-                if not categories:
-                    validation_result.add_error(
-                        f"Categorical distribution for field '{field_name}' missing categories",
-                        "Categorical distributions require category parameters"
-                    )
-            
-            elif distribution.strategy_type == DistributionType.NUMERIC_RANGE:
-                min_val = distribution.parameters.get('min')
-                max_val = distribution.parameters.get('max')
-                if min_val is None or max_val is None:
-                    validation_result.add_error(
-                        f"Numeric range distribution for field '{field_name}' missing range parameters",
-                        "Numeric range distributions require min and max parameters"
-                    )
-                elif min_val >= max_val:
-                    validation_result.add_error(
-                        f"Invalid range for field '{field_name}': min must be less than max",
-                        f"Current range: {min_val} to {max_val}"
-                    )
-        
-        except Exception as e:
-            validation_result.add_error(
-                f"Error validating distribution for field '{field_name}': {e}",
-                "Check distribution parameters"
-            )
-        
-        return validation_result
-    
-    def _convert_to_field_definition(self, parsed_field: ParsedFieldDefinition,
-                                   distribution: Optional[DistributionStrategy] = None) -> FieldDefinition:
-        """
-        Convert parsed field definition to FieldDefinition model.
-        
-        Enhanced to handle new pattern structure with pattern_value and pattern_type.
-        
-        Args:
-            parsed_field: Parsed field definition
-            distribution: Optional distribution strategy
-            
-        Returns:
-            FieldDefinition object
-        """
-        # For reference patterns, use the pattern_value as pattern_name
-        # For inline patterns, we'll need to handle them differently in the generator
-        pattern_name = parsed_field.pattern_value if parsed_field.pattern_type == "reference" else parsed_field.name
-        
-        return FieldDefinition(
-            name=parsed_field.name,
-            pattern_name=pattern_name,
-            distribution=distribution,
-            constraints=parsed_field.constraints
-        )
-    
-    def _is_valid_field_name(self, name: str) -> bool:
-        """Check if a field name is valid."""
-        import keyword
-        return (name and 
-                name.isidentifier() and 
-                not name.startswith('_') and 
-                not keyword.iskeyword(name) and
-                name.replace('_', '').isalnum())
-    
-    def _is_valid_pattern_name(self, name: str) -> bool:
-        """Check if a pattern name is valid."""
-        import keyword
-        return (name and 
-                name.isidentifier() and 
-                not name.startswith('_') and 
-                not keyword.iskeyword(name) and
-                name.replace('_', '').isalnum())
-    
-    def _is_valid_import_name(self, name: str) -> bool:
-        """Check if an import name is valid."""
-        return (name and 
-                name.replace('_', '').replace('-', '').isalnum() and
-                not name.startswith('_') and
-                len(name) <= 50)
-    
-    def clear_cache(self):
-        """Clear all caches."""
-        self.schema_file_manager.clear_cache()
-        self.pattern_resolver.clear_cache()