additory 0.1.0a1__py3-none-any.whl → 0.1.0a3__py3-none-any.whl

additory/synthetic/generator_parser.py

@@ -1,68 +0,0 @@
-#!/usr/bin/env python3
-"""
-Generator parser for synthetic data generation
-Handles .gen file parsing and generator resolution
-"""
-
-import os
-from typing import Dict, List
-
-BASE = os.path.join(os.path.dirname(__file__), "..", "..", "reference", "schema_definitions")
-
-def load_gen_file(filename: str) -> Dict[str, str]:
-    """
-    Load generator definitions from a .gen file
-    
-    Args:
-        filename: Name of the .gen file
-        
-    Returns:
-        Dictionary mapping generator names to expressions
-    """
-    full_path = os.path.join(BASE, filename)
-    generators = {}
-    
-    if not os.path.exists(full_path):
-        return generators
-    
-    try:
-        with open(full_path, "r") as f:
-            for line in f:
-                line = line.strip()
-                if line and ":" in line and not line.startswith("#"):
-                    key, expr = line.split(":", 1)
-                    generators[key.strip()] = expr.strip()
-    except Exception:
-        pass
-    
-    return generators
-
-
-def resolve_generators(imports: List[str], inline_gens: Dict[str, str]) -> Dict[str, str]:
-    """
-    Resolve generators from imports and inline definitions
-    
-    Args:
-        imports: List of .gen files to import
-        inline_gens: Inline generator definitions
-        
-    Returns:
-        Combined dictionary of all generators
-    """
-    generators = {}
-    
-    # Load global.gen first (lowest priority)
-    global_gens = load_gen_file("global.gen")
-    generators.update(global_gens)
-    
-    # Load imported .gen files (medium priority)
-    for import_file in imports:
-        if not import_file.endswith(".gen"):
-            import_file += ".gen"
-        imported_gens = load_gen_file(import_file)
-        generators.update(imported_gens)
-    
-    # Apply inline generators (highest priority)
-    generators.update(inline_gens)
-    
-    return generators

additory/synthetic/integration.py

@@ -1,319 +0,0 @@
-"""
-Integration layer for synthetic data generation with fail-fast validation.
-
-Provides high-level interfaces that integrate all components with comprehensive
-validation and clear error reporting.
-"""
-
-from typing import Dict, List, Optional, Union
-from pathlib import Path
-import os
-
-from .models import (
-    SchemaDefinition,
-    GenerationContext,
-    GenerationResult,
-    ResolvedPattern,
-    ValidationResult
-)
-from .file_managers import SchemaFileManager, PatternFileManager, ParsedSchemaFile
-from .pattern_resolver import PatternHierarchyResolver
-from .generator import SyntheticDataGenerator, GenerationConfig
-from .validator import ValidationSystem
-from .exceptions import ValidationError, SyntheticDataError
-
-
-class SyntheticDataIntegrator:
-    """High-level integrator for synthetic data generation with validation."""
-    
-    def __init__(self, config: Optional[GenerationConfig] = None):
-        """Initialize the integrator with all components."""
-        self.config = config or GenerationConfig()
-        self.validation_system = ValidationSystem()
-        self.schema_manager = SchemaFileManager()
-        self.pattern_manager = PatternFileManager()
-        self.pattern_resolver = PatternHierarchyResolver()
-        self.generator = SyntheticDataGenerator(self.config)
-    
-    def generate_from_schema_file(self, 
-                                 schema_path: str,
-                                 target_rows: int,
-                                 output_engine: str = "pandas") -> GenerationResult:
-        """
-        Generate synthetic data from a schema file with comprehensive validation.
-        
-        This is the main entry point that performs:
-        1. File validation
-        2. Schema parsing and validation
-        3. Pattern resolution and validation
-        4. Fail-fast validation before generation
-        5. Data generation
-        
-        Args:
-            schema_path: Path to the .toml schema file
-            target_rows: Number of rows to generate
-            output_engine: Output format ("pandas" or "polars")
-            
-        Returns:
-            GenerationResult with generated data and metadata
-            
-        Raises:
-            ValidationError: If any validation fails (fail-fast)
-            SyntheticDataError: If generation fails
-        """
-        # Step 1: Validate file exists and has correct extension
-        self._validate_schema_file_path(schema_path)
-        
-        # Step 2: Validate file syntax
-        file_validation = self.validation_system.validate_file_syntax(schema_path)
-        if not file_validation.is_valid:
-            self._raise_validation_error("Schema file validation failed", file_validation)
-        
-        # Step 3: Load and parse schema
-        try:
-            parsed_schema = self.schema_manager.load_toml_schema(schema_path)
-            schema = self._convert_parsed_schema_to_definition(parsed_schema)
-            schema.source_file = schema_path
-        except Exception as e:
-            raise SyntheticDataError(f"Failed to load schema file '{schema_path}': {e}")
-        
-        # Step 4: Resolve all patterns
-        try:
-            resolved_patterns = self._resolve_schema_patterns(schema)
-        except Exception as e:
-            raise SyntheticDataError(f"Failed to resolve patterns: {e}")
-        
-        # Step 5: Create generation context
-        context = GenerationContext(
-            schema=schema,
-            resolved_patterns=resolved_patterns,
-            target_rows=target_rows,
-            output_engine=output_engine,
-            seed=self.config.seed
-        )
-        
-        # Step 6: Comprehensive fail-fast validation (if enabled)
-        if self.config.validate_patterns:
-            self.validation_system.validate_fail_fast(
-                schema, resolved_patterns, schema_path
-            )
-        
-        # Step 7: Generate data (validation already passed)
-        return self.generator.generate(context)
-    
-    def validate_schema_file(self, schema_path: str) -> ValidationResult:
-        """
-        Validate a schema file without generating data.
-        
-        Performs comprehensive validation including:
-        - File syntax validation
-        - Schema structure validation
-        - Pattern resolution validation
-        - Distribution strategy validation
-        
-        Args:
-            schema_path: Path to the .toml schema file
-            
-        Returns:
-            ValidationResult with all validation issues
-        """
-        result = ValidationResult(is_valid=True)
-        
-        try:
-            # Validate file path
-            self._validate_schema_file_path(schema_path)
-            
-            # Validate file syntax
-            file_validation = self.validation_system.validate_file_syntax(schema_path)
-            result.merge(file_validation)
-            
-            if not file_validation.is_valid:
-                return result  # Can't continue if file syntax is invalid
-            
-            # Load schema
-            schema = self.schema_manager.load_schema_file(schema_path)
-            schema.source_file = schema_path
-            
-            # Resolve patterns
-            resolved_patterns = self._resolve_schema_patterns(schema)
-            
-            # Comprehensive validation
-            comprehensive_validation = self.validation_system.validate_complete_configuration(
-                schema, resolved_patterns, schema_path
-            )
-            result.merge(comprehensive_validation)
-            
-        except Exception as e:
-            result.add_error(f"Validation failed: {e}")
-        
-        return result
-    
-    def validate_pattern_file(self, pattern_path: str) -> ValidationResult:
-        """
-        Validate a pattern file (.properties).
-        
-        Args:
-            pattern_path: Path to the .properties file
-            
-        Returns:
-            ValidationResult with validation issues
-        """
-        return self.validation_system.validate_file_syntax(pattern_path)
-    
-    def get_available_patterns(self, search_paths: Optional[List[str]] = None) -> Dict[str, str]:
-        """
-        Get all available patterns from the pattern hierarchy.
-        
-        Args:
-            search_paths: Optional list of paths to search for patterns
-            
-        Returns:
-            Dictionary mapping pattern names to their regex definitions
-        """
-        if search_paths is None:
-            search_paths = self._get_default_pattern_paths()
-        
-        all_patterns = {}
-        
-        for path in search_paths:
-            if os.path.exists(path) and path.endswith('.properties'):
-                try:
-                    patterns = self.pattern_manager.load_properties_file(path)
-                    all_patterns.update(patterns)
-                except Exception:
-                    continue  # Skip files that can't be loaded
-        
-        return all_patterns
-    
-    def _validate_schema_file_path(self, schema_path: str) -> None:
-        """Validate schema file path and extension."""
-        if not os.path.exists(schema_path):
-            raise ValidationError(f"Schema file not found: {schema_path}")
-        
-        if not schema_path.endswith('.toml'):
-            raise ValidationError(
-                f"Invalid schema file extension. Expected .toml, got: {Path(schema_path).suffix}"
-            )
-    
-    def _resolve_schema_patterns(self, schema: SchemaDefinition) -> Dict[str, ResolvedPattern]:
-        """Resolve all patterns referenced in the schema."""
-        pattern_names = schema.get_all_pattern_names()
-        
-        try:
-            resolution_results = self.pattern_resolver.resolve_multiple_patterns(
-                pattern_names,
-                inline_patterns=schema.inline_patterns,
-                user_imports=schema.imports
-            )
-            
-            # Convert PatternResolutionResult to ResolvedPattern
-            resolved_patterns = {}
-            for name, result in resolution_results.items():
-                # Extract resolution chain from search order
-                resolution_chain = [source for source, _, _ in result.trace.search_order]
-                
-                resolved_pattern = ResolvedPattern(
-                    definition=result.pattern,
-                    resolution_chain=resolution_chain,
-                    final_source=result.trace.resolved_source
-                )
-                resolved_patterns[name] = resolved_pattern
-            
-            return resolved_patterns
-            
-        except Exception as e:
-            raise SyntheticDataError(f"Pattern resolution failed: {e}")
-    
-    def _convert_parsed_schema_to_definition(self, parsed_schema: ParsedSchemaFile) -> SchemaDefinition:
-        """Convert ParsedSchemaFile to SchemaDefinition."""
-        from .models import FieldDefinition, DistributionStrategy, DistributionType
-        
-        schema = SchemaDefinition(
-            imports=parsed_schema.imports,
-            inline_patterns=parsed_schema.inline_patterns,
-            metadata=parsed_schema.metadata
-        )
-        
-        # Convert schema definitions to field definitions
-        # For now, create simple field definitions - this will be enhanced in later tasks
-        for field_name, pattern_name in parsed_schema.schema_definitions.items():
-            # Create a simple equal distribution for now
-            distribution = DistributionStrategy(
-                strategy_type=DistributionType.EQUAL,
-                validate_on_init=False
-            )
-            
-            field_def = FieldDefinition(
-                name=field_name,
-                pattern_name=pattern_name,
-                distribution=distribution
-            )
-            
-            schema.add_field(field_def)
-        
-        return schema
-    
-    def _get_default_pattern_paths(self) -> List[str]:
-        """Get default paths to search for pattern files."""
-        paths = []
-        
-        # Global patterns
-        global_path = os.path.join("reference", "schema_definitions", "global.properties")
-        if os.path.exists(global_path):
-            paths.append(global_path)
-        
-        # User patterns (if they exist)
-        user_global = os.path.expanduser("~/.additory/patterns/global.properties")
-        if os.path.exists(user_global):
-            paths.append(user_global)
-        
-        return paths
-    
-    def _raise_validation_error(self, message: str, validation_result: ValidationResult) -> None:
-        """Raise a ValidationError with formatted validation results."""
-        error_details = []
-        
-        for error in validation_result.errors:
-            error_details.append(f"  ERROR: {error}")
-        
-        for warning in validation_result.warnings:
-            error_details.append(f"  WARNING: {warning}")
-        
-        full_message = f"{message}:\n" + "\n".join(error_details)
-        raise ValidationError(full_message)
-
-
-class ValidationHelper:
-    """Helper class for validation operations and error reporting."""
-    
-    @staticmethod
-    def format_validation_report(result: ValidationResult, title: str = "Validation Report") -> str:
-        """Format a validation result into a human-readable report."""
-        lines = [f"=== {title} ===", ""]
-        
-        if result.is_valid:
-            lines.append("✅ All validations passed!")
-            if result.warnings:
-                lines.append(f"\n⚠️  {len(result.warnings)} warning(s):")
-                for warning in result.warnings:
-                    lines.append(f"  • {warning}")
-        else:
-            lines.append(f"❌ Validation failed with {len(result.errors)} error(s):")
-            for error in result.errors:
-                lines.append(f"  • {error}")
-            
-            if result.warnings:
-                lines.append(f"\n⚠️  {len(result.warnings)} warning(s):")
-                for warning in result.warnings:
-                    lines.append(f"  • {warning}")
-        
-        return "\n".join(lines)
-    
-    @staticmethod
-    def get_validation_summary(result: ValidationResult) -> Dict[str, int]:
-        """Get a summary of validation results."""
-        return {
-            "errors": len(result.errors),
-            "warnings": len(result.warnings),
-            "is_valid": result.is_valid
-        }

additory/synthetic/models.py

@@ -1,241 +0,0 @@
-"""
-Core data models for synthetic data generation system.
-
-Defines the data structures used throughout the system for patterns,
-distributions, schemas, and generation contexts.
-"""
-
-from dataclasses import dataclass, field
-from enum import Enum
-from typing import Dict, List, Any, Optional, Union
-import polars as pl
-
-
-class PatternSource(Enum):
-    """Sources for pattern definitions in the hierarchy."""
-    INLINE = "inline"
-    USER_IMPORT = "user_import"
-    USER_GLOBAL = "user_global"
-    CORE_NON_GLOBAL = "core_non_global"
-    CORE_GLOBAL = "core_global"
-
-
-class ValidationStatus(Enum):
-    """Validation status for patterns and schemas."""
-    VALID = "valid"
-    INVALID = "invalid"
-    PENDING = "pending"
-    NOT_VALIDATED = "not_validated"
-
-
-class DistributionType(Enum):
-    """Supported distribution strategy types."""
-    EQUAL = "equal"
-    CUSTOM = "custom"
-    CATEGORICAL = "categorical"
-    HIGH_CARDINALITY = "high_cardinality"
-    NUMERIC_RANGE = "numeric_range"
-    SKEWED = "skewed"
-
-
-@dataclass
-class PatternDefinition:
-    """Represents a regex pattern definition with metadata."""
-    name: str
-    regex: str
-    source: PatternSource
-    validation_status: ValidationStatus = ValidationStatus.NOT_VALIDATED
-    polars_compatible: bool = False
-    source_file: Optional[str] = None
-    line_number: Optional[int] = None
-    
-    def __post_init__(self):
-        """Validate pattern definition after initialization."""
-        if not self.name:
-            raise ValueError("Pattern name cannot be empty")
-        if not self.regex:
-            raise ValueError("Pattern regex cannot be empty")
-
-
-@dataclass
-class ValidationRule:
-    """Represents a validation rule for distributions."""
-    rule_type: str
-    parameters: Dict[str, Any]
-    error_message: str
-
-
-@dataclass
-class DistributionStrategy:
-    """Represents a distribution strategy configuration."""
-    strategy_type: DistributionType
-    parameters: Dict[str, Any] = field(default_factory=dict)
-    validation_rules: List[ValidationRule] = field(default_factory=list)
-    validate_on_init: bool = True
-    
-    def __post_init__(self):
-        """Validate distribution strategy after initialization."""
-        if self.validate_on_init:
-            self._validate_parameters()
-    
-    def _validate_parameters(self):
-        """Validate parameters based on strategy type."""
-        if self.strategy_type == DistributionType.CUSTOM:
-            if 'weights' not in self.parameters:
-                raise ValueError("Custom distribution requires 'weights' parameter")
-        elif self.strategy_type == DistributionType.NUMERIC_RANGE:
-            if 'min' not in self.parameters or 'max' not in self.parameters:
-                raise ValueError("Numeric range distribution requires 'min' and 'max' parameters")
-        elif self.strategy_type == DistributionType.CATEGORICAL:
-            if 'categories' not in self.parameters:
-                raise ValueError("Categorical distribution requires 'categories' parameter")
-
-
-@dataclass
-class FieldDefinition:
-    """Represents a field definition in a schema."""
-    name: str
-    pattern_name: str
-    distribution: DistributionStrategy
-    constraints: List[str] = field(default_factory=list)
-    
-    def __post_init__(self):
-        """Validate field definition after initialization."""
-        if not self.name:
-            raise ValueError("Field name cannot be empty")
-        if not self.pattern_name:
-            raise ValueError("Pattern name cannot be empty")
-
-
-@dataclass
-class SchemaDefinition:
-    """Represents a complete schema definition."""
-    imports: List[str] = field(default_factory=list)
-    inline_patterns: Dict[str, str] = field(default_factory=dict)
-    field_definitions: Dict[str, FieldDefinition] = field(default_factory=dict)
-    metadata: Dict[str, Any] = field(default_factory=dict)
-    source_file: Optional[str] = None
-    
-    def add_field(self, field_def: FieldDefinition):
-        """Add a field definition to the schema."""
-        self.field_definitions[field_def.name] = field_def
-    
-    def get_field(self, name: str) -> Optional[FieldDefinition]:
-        """Get a field definition by name."""
-        return self.field_definitions.get(name)
-    
-    def get_all_pattern_names(self) -> List[str]:
-        """Get all pattern names referenced in the schema."""
-        pattern_names = list(self.inline_patterns.keys())
-        for field_def in self.field_definitions.values():
-            if field_def.pattern_name not in pattern_names:
-                pattern_names.append(field_def.pattern_name)
-        return pattern_names
-
-
-@dataclass
-class ResolvedPattern:
-    """Represents a pattern that has been resolved through the hierarchy."""
-    definition: PatternDefinition
-    resolution_chain: List[PatternSource]
-    final_source: PatternSource
-    
-    @property
-    def name(self) -> str:
-        return self.definition.name
-    
-    @property
-    def regex(self) -> str:
-        return self.definition.regex
-    
-    @property
-    def is_valid(self) -> bool:
-        return (self.definition.validation_status == ValidationStatus.VALID and 
-                self.definition.polars_compatible)
-
-
-@dataclass
-class GenerationContext:
-    """Context for data generation operations."""
-    schema: SchemaDefinition
-    resolved_patterns: Dict[str, ResolvedPattern]
-    target_rows: int
-    output_engine: str = "pandas"
-    batch_size: int = 10000
-    seed: Optional[int] = None
-    
-    def get_pattern(self, name: str) -> Optional[ResolvedPattern]:
-        """Get a resolved pattern by name."""
-        return self.resolved_patterns.get(name)
-    
-    def validate_context(self) -> bool:
-        """Validate that the generation context is complete and valid."""
-        # Check that all required patterns are resolved
-        for field_def in self.schema.field_definitions.values():
-            if field_def.pattern_name not in self.resolved_patterns:
-                return False
-            
-            pattern = self.resolved_patterns[field_def.pattern_name]
-            if not pattern.is_valid:
-                return False
-        
-        return True
-
-
-@dataclass
-class GenerationResult:
-    """Result of a data generation operation."""
-    dataframe: Union[pl.DataFrame, 'pd.DataFrame']
-    metadata: Dict[str, Any] = field(default_factory=dict)
-    generation_time: Optional[float] = None
-    memory_usage: Optional[int] = None
-    
-    @property
-    def row_count(self) -> int:
-        """Get the number of rows in the generated dataframe."""
-        if hasattr(self.dataframe, 'height'):  # polars
-            return self.dataframe.height
-        else:  # pandas
-            return len(self.dataframe)
-    
-    @property
-    def column_count(self) -> int:
-        """Get the number of columns in the generated dataframe."""
-        if hasattr(self.dataframe, 'width'):  # polars
-            return self.dataframe.width
-        else:  # pandas
-            return len(self.dataframe.columns)
-
-
-@dataclass
-class ValidationResult:
-    """Result of a validation operation."""
-    is_valid: bool
-    errors: List[str] = field(default_factory=list)
-    warnings: List[str] = field(default_factory=list)
-    suggestions: List[str] = field(default_factory=list)
-    
-    def add_error(self, error: str, suggestion: Optional[str] = None):
-        """Add an error to the validation result."""
-        self.errors.append(error)
-        self.is_valid = False
-        if suggestion:
-            self.suggestions.append(suggestion)
-    
-    def add_warning(self, warning: str):
-        """Add a warning to the validation result."""
-        self.warnings.append(warning)
-    
-    def merge(self, other: 'ValidationResult'):
-        """Merge another validation result into this one."""
-        self.errors.extend(other.errors)
-        self.warnings.extend(other.warnings)
-        self.suggestions.extend(other.suggestions)
-        if not other.is_valid:
-            self.is_valid = False
-
-
-# Type aliases for common types
-PatternDict = Dict[str, PatternDefinition]
-ResolvedPatternDict = Dict[str, ResolvedPattern]
-DistributionDict = Dict[str, DistributionStrategy]