additory 0.1.0a1__py3-none-any.whl

additory/synthetic/api.py

@@ -0,0 +1,220 @@
+"""
+Main API interface for synthetic data generation.
+
+Provides the primary user-facing functions for generating synthetic data
+with support for different output engines and configuration management.
+"""
+
+from typing import Union, Optional, Type
+import pandas as pd
+import polars as pl
+import os
+from pathlib import Path
+
+from .config import SyntheticConfig
+from .exceptions import SyntheticDataError, ValidationError
+from .integration import SyntheticDataIntegrator
+from .generator import GenerationConfig
+from .engines import DistributionEngineFactory, DistributionEngine
+
+
+# Global configuration instance
+config = SyntheticConfig()
+
+
+def synth(schema_path: str, rows: int = 1000, 
+          engine: Optional[str] = None) -> Union[pd.DataFrame, pl.DataFrame]:
+    """
+    Generate synthetic data from a schema file.
+    
+    Args:
+        schema_path: Path to the .toml schema file
+        rows: Number of rows to generate (default: 1000)
+        engine: Output engine ("pandas" or "polars"). If None, uses default from config
+        
+    Returns:
+        Generated DataFrame in the specified format
+        
+    Raises:
+        SyntheticDataError: If generation fails
+        ValidationError: If schema validation fails
+        FileNotFoundError: If schema file doesn't exist
+        
+    Examples:
+        >>> df = synth("customer.toml", rows=5000)  # pandas DataFrame
+        >>> df = synth("customer.toml", rows=5000, engine="polars")  # polars DataFrame
+    """
+    # Validate inputs
+    if rows <= 0:
+        raise ValueError("Number of rows must be positive")
+    
+    # Determine output engine
+    output_engine = engine if engine is not None else config.get_default_engine()
+    if output_engine not in ["pandas", "polars"]:
+        raise ValueError(f"Unsupported engine: {output_engine}. Must be 'pandas' or 'polars'")
+    
+    # Resolve schema path if it's relative to the configured base path
+    resolved_schema_path = _resolve_schema_path(schema_path)
+    
+    # Create generation configuration
+    generation_config = GenerationConfig(
+        batch_size=config.get_default_batch_size(),
+        seed=None,  # Use random seed by default
+        validate_patterns=config.is_validation_enabled()
+    )
+    
+    # Create integrator and generate data
+    integrator = SyntheticDataIntegrator(generation_config)
+    
+    try:
+        result = integrator.generate_from_schema_file(
+            schema_path=resolved_schema_path,
+            target_rows=rows,
+            output_engine=output_engine
+        )
+        return result.dataframe
+        
+    except ValidationError:
+        # Re-raise validation errors as-is
+        raise
+    except Exception as e:
+        # Wrap other exceptions in SyntheticDataError
+        raise SyntheticDataError(f"Failed to generate synthetic data: {e}") from e
+
+
+def register_distribution_engine(engine_class: Type[DistributionEngine]) -> None:
+    """
+    Register a custom distribution engine for use in synthetic data generation.
+    
+    Custom engines allow you to implement your own distribution strategies beyond
+    the built-in ones (equal, custom, categorical, high_cardinality, numeric_range, skewed).
+    
+    Args:
+        engine_class: A class that inherits from DistributionEngine and implements
+                     the required methods (supports_strategy, validate_config, apply_distribution)
+        
+    Raises:
+        ValidationError: If the engine class is invalid or already registered
+        
+    Examples:
+        >>> from additory.synthetic.engines import DistributionEngine, DistributionConfig
+        >>> from additory.synthetic.models import DistributionType, ValidationResult
+        >>> import polars as pl
+        >>> 
+        >>> class GaussianDistributionEngine(DistributionEngine):
+        ...     def supports_strategy(self, strategy_type):
+        ...         return strategy_type == DistributionType.CUSTOM and strategy_type.value == "gaussian"
+        ...     
+        ...     def validate_config(self, config: DistributionConfig) -> ValidationResult:
+        ...         result = self._validate_base_requirements(config)
+        ...         # Add custom validation
+        ...         return result
+        ...     
+        ...     def apply_distribution(self, config: DistributionConfig) -> pl.Series:
+        ...         # Implement gaussian distribution logic
+        ...         pass
+        >>> 
+        >>> register_distribution_engine(GaussianDistributionEngine)
+    """
+    DistributionEngineFactory.register_custom_engine(engine_class)
+
+
+def unregister_distribution_engine(engine_class: Type[DistributionEngine]) -> None:
+    """
+    Unregister a previously registered custom distribution engine.
+    
+    Args:
+        engine_class: The engine class to unregister
+        
+    Raises:
+        ValidationError: If the engine is not registered
+        
+    Examples:
+        >>> unregister_distribution_engine(GaussianDistributionEngine)
+    """
+    DistributionEngineFactory.unregister_custom_engine(engine_class)
+
+
+def list_custom_distribution_engines() -> list:
+    """
+    Get a list of all registered custom distribution engines.
+    
+    Returns:
+        List of custom engine classes
+        
+    Examples:
+        >>> engines = list_custom_distribution_engines()
+        >>> for engine in engines:
+        ...     print(engine.__name__)
+    """
+    return DistributionEngineFactory.list_custom_engines()
+
+
+def _resolve_schema_path(schema_path: str) -> str:
+    """
+    Resolve schema path, checking both absolute and relative to config base path.
+    
+    Args:
+        schema_path: Input schema path
+        
+    Returns:
+        Resolved absolute path to schema file
+        
+    Raises:
+        FileNotFoundError: If schema file cannot be found
+    """
+    # If it's an absolute path or exists as-is, use it directly
+    if os.path.isabs(schema_path) or os.path.exists(schema_path):
+        if os.path.exists(schema_path):
+            return schema_path
+        else:
+            raise FileNotFoundError(f"Schema file not found: {schema_path}")
+    
+    # Try resolving relative to the configured base path
+    resolved_path = config.resolve_schema_path(schema_path)
+    if resolved_path.exists():
+        return str(resolved_path)
+    
+    # Try current working directory
+    cwd_path = Path.cwd() / schema_path
+    if cwd_path.exists():
+        return str(cwd_path)
+    
+    # File not found in any location
+    raise FileNotFoundError(
+        f"Schema file not found: {schema_path}. "
+        f"Searched in: current directory, {config.get_schema_base_path()}"
+    )
+
+
+def augment(df: Union[pd.DataFrame, pl.DataFrame], 
+           schema_path: str, **kwargs) -> Union[pd.DataFrame, pl.DataFrame]:
+    """
+    Augment existing DataFrame with synthetic columns.
+    
+    This function will be implemented in future phases to support data augmentation
+    for class balancing and other use cases.
+    
+    Args:
+        df: Input DataFrame to augment
+        schema_path: Path to the .toml schema file
+        **kwargs: Additional augmentation parameters
+        
+    Returns:
+        Augmented DataFrame in the same format as input
+        
+    Raises:
+        NotImplementedError: This feature is planned for future implementation
+    """
+    raise NotImplementedError("augment() function is planned for future implementation")
+
+
+# Export the config object and plugin functions for user access
+__all__ = [
+    'synth', 
+    'augment', 
+    'config',
+    'register_distribution_engine',
+    'unregister_distribution_engine',
+    'list_custom_distribution_engines'
+]

additory/synthetic/common_integration.py

@@ -0,0 +1,314 @@
+"""
+Common Module Integration for Synthetic Data Generation
+
+Provides integration between the common module (lists, patterns, resolver)
+and the synthetic data generation system.
+
+This module:
+- Wraps common/resolver.py for synthetic-specific needs
+- Handles .list and .properties file loading
+- Implements prefer_mode logic
+- Provides pattern type detection
+- Maintains backward compatibility
+"""
+
+from typing import Union, List, Optional, Dict, Any
+from pathlib import Path
+import logging
+
+from additory.common.resolver import (
+    PatternResolver,
+    resolve_pattern,
+    PreferMode,
+    PatternResolutionResult,
+)
+from additory.common.lists import load_list_file, ListFileError
+from additory.common.patterns import load_properties_file, is_regex_pattern, PatternFileError
+
+from .exceptions import PatternResolutionError, ValidationError
+
+
+logger = logging.getLogger(__name__)
+
+
+class SyntheticPatternLoader:
+    """
+    Pattern loader for synthetic data generation.
+    
+    Integrates with common module to provide:
+    - .list file loading
+    - .properties file loading
+    - Fallback resolution
+    - Pattern type detection
+    - Prefer mode support
+    """
+    
+    def __init__(self, base_path: str = "reference/schema_definitions"):
+        """
+        Initialize the pattern loader.
+        
+        Args:
+            base_path: Base directory for pattern files
+        """
+        self.base_path = Path(base_path)
+        self.resolver = PatternResolver(base_path=str(self.base_path))
+        
+        # Cache for loaded files
+        self._list_cache: Dict[str, Dict[str, List[str]]] = {}
+        self._properties_cache: Dict[str, Dict[str, str]] = {}
+    
+    def load_pattern(
+        self,
+        pattern_value: Union[str, List[str]],
+        imports: List[str],
+        prefer_mode: str = "default"
+    ) -> tuple[Union[List[str], str], str]:
+        """
+        Load a pattern with fallback resolution.
+        
+        Args:
+            pattern_value: Pattern value from TOML (string reference, array list, or inline regex)
+            imports: List of imports from TOML (e.g., ["global", "finance"])
+            prefer_mode: Resolution preference ("default", "list_only", "regex_only")
+            
+        Returns:
+            Tuple of (resolved_value, pattern_type)
+            - resolved_value: List of values (for lists) or regex string (for regex)
+            - pattern_type: "list" or "regex"
+            
+        Raises:
+            PatternResolutionError: If pattern cannot be resolved
+            
+        Example:
+            >>> loader = SyntheticPatternLoader()
+            >>> 
+            >>> # Reference (resolved via fallback)
+            >>> value, type = loader.load_pattern("first_names", ["global"], "default")
+            >>> # value = ['Arjun', 'Vikram', ...], type = 'list'
+            >>> 
+            >>> # Inline list
+            >>> value, type = loader.load_pattern(["Active", "Inactive"], ["global"], "default")
+            >>> # value = ['Active', 'Inactive'], type = 'list'
+            >>> 
+            >>> # Inline regex
+            >>> value, type = loader.load_pattern("CUST\\d{8}", ["global"], "default")
+            >>> # value = 'CUST\\d{8}', type = 'regex'
+        """
+        # Detect pattern type
+        pattern_type = self._detect_pattern_type(pattern_value)
+        
+        if pattern_type == "inline_list":
+            # Inline list (array)
+            logger.info(f"Using inline list with {len(pattern_value)} values")
+            return (pattern_value, "list")
+        
+        elif pattern_type == "inline_regex":
+            # Inline regex (string with special chars)
+            logger.info(f"Using inline regex: {pattern_value}")
+            return (pattern_value, "regex")
+        
+        elif pattern_type == "reference":
+            # Reference (resolve via fallback)
+            logger.info(f"Resolving reference: {pattern_value}")
+            return self._resolve_reference(pattern_value, imports, prefer_mode)
+        
+        else:
+            raise PatternResolutionError(
+                f"Unknown pattern type for value: {pattern_value}"
+            )
+    
+    def _detect_pattern_type(self, pattern_value: Union[str, List[str]]) -> str:
+        """
+        Detect pattern type from value.
+        
+        Args:
+            pattern_value: Pattern value from TOML
+            
+        Returns:
+            Pattern type: "inline_list", "inline_regex", or "reference"
+        """
+        if isinstance(pattern_value, list):
+            # Array = inline list
+            return "inline_list"
+        
+        elif isinstance(pattern_value, str):
+            # String: check if it's regex or reference
+            if is_regex_pattern(pattern_value):
+                # Has special regex chars = inline regex
+                return "inline_regex"
+            else:
+                # Simple string = reference
+                return "reference"
+        
+        else:
+            raise ValidationError(
+                f"Invalid pattern value type: {type(pattern_value)}. "
+                f"Expected string or list."
+            )
+    
+    def _resolve_reference(
+        self,
+        pattern_name: str,
+        imports: List[str],
+        prefer_mode: str
+    ) -> tuple[Union[List[str], str], str]:
+        """
+        Resolve a pattern reference using fallback resolution.
+        
+        Args:
+            pattern_name: Name of pattern to resolve
+            imports: List of imports
+            prefer_mode: Resolution preference
+            
+        Returns:
+            Tuple of (resolved_value, pattern_type)
+            
+        Raises:
+            PatternResolutionError: If pattern cannot be resolved
+        """
+        # Convert prefer_mode string to enum
+        try:
+            mode = PreferMode(prefer_mode)
+        except ValueError:
+            logger.warning(f"Invalid prefer_mode '{prefer_mode}', using DEFAULT")
+            mode = PreferMode.DEFAULT
+        
+        # Resolve using common module
+        result = self.resolver.resolve(pattern_name, imports, mode)
+        
+        if not result.found:
+            raise PatternResolutionError(
+                f"Pattern '{pattern_name}' not found. {result.error_message}",
+                pattern_name,
+                imports
+            )
+        
+        # Log resolution
+        logger.info(
+            f"Resolved '{pattern_name}' from {result.source} "
+            f"(type: {result.pattern_type}, fallback: {result.fallback_used})"
+        )
+        
+        return (result.value, result.pattern_type)
+    
+    def validate_imports(self, imports: List[str]) -> tuple[bool, List[str]]:
+        """
+        Validate that import files exist.
+        
+        Args:
+            imports: List of import names (e.g., ["global", "finance"])
+            
+        Returns:
+            Tuple of (is_valid, error_messages)
+        """
+        errors = []
+        
+        for import_name in imports:
+            # Check for .list file
+            list_file = self.base_path / f"{import_name}.list"
+            properties_file = self.base_path / f"{import_name}.properties"
+            
+            # At least one should exist
+            if not list_file.exists() and not properties_file.exists():
+                errors.append(
+                    f"Import '{import_name}' not found. "
+                    f"Neither {import_name}.list nor {import_name}.properties exists."
+                )
+        
+        return (len(errors) == 0, errors)
+    
+    def get_available_patterns(self, imports: List[str]) -> Dict[str, str]:
+        """
+        Get all available patterns from imports.
+        
+        Args:
+            imports: List of import names
+            
+        Returns:
+            Dictionary mapping pattern names to sources
+        """
+        available = {}
+        
+        for import_name in imports:
+            # Load .list file if exists
+            list_file = self.base_path / f"{import_name}.list"
+            if list_file.exists():
+                try:
+                    lists = load_list_file(str(list_file))
+                    for list_name in lists.keys():
+                        available[list_name] = f"{import_name}.list"
+                except ListFileError as e:
+                    logger.warning(f"Failed to load {list_file}: {e}")
+            
+            # Load .properties file if exists
+            properties_file = self.base_path / f"{import_name}.properties"
+            if properties_file.exists():
+                try:
+                    patterns = load_properties_file(str(properties_file))
+                    for pattern_name in patterns.keys():
+                        # Don't overwrite if already in list
+                        if pattern_name not in available:
+                            available[pattern_name] = f"{import_name}.properties"
+                except PatternFileError as e:
+                    logger.warning(f"Failed to load {properties_file}: {e}")
+        
+        return available
+    
+    def clear_cache(self):
+        """Clear cached files (useful for testing)."""
+        self._list_cache.clear()
+        self._properties_cache.clear()
+        self.resolver.clear_cache()
+
+
+def detect_pattern_type_from_toml(pattern_value: Any) -> str:
+    """
+    Detect pattern type from TOML value.
+    
+    This is a convenience function for use in schema parsing.
+    
+    Args:
+        pattern_value: Value from TOML file
+        
+    Returns:
+        Pattern type: "inline_list", "inline_regex", or "reference"
+        
+    Example:
+        >>> detect_pattern_type_from_toml(["Active", "Inactive"])
+        'inline_list'
+        >>> detect_pattern_type_from_toml("CUST\\d{8}")
+        'inline_regex'
+        >>> detect_pattern_type_from_toml("first_names")
+        'reference'
+    """
+    if isinstance(pattern_value, list):
+        return "inline_list"
+    elif isinstance(pattern_value, str):
+        if is_regex_pattern(pattern_value):
+            return "inline_regex"
+        else:
+            return "reference"
+    else:
+        return "unknown"
+
+
+def convert_prefer_mode(mode_str: str) -> PreferMode:
+    """
+    Convert prefer_mode string to enum.
+    
+    Args:
+        mode_str: Mode string ("default", "list_only", "regex_only")
+        
+    Returns:
+        PreferMode enum value
+        
+    Raises:
+        ValueError: If mode string is invalid
+    """
+    try:
+        return PreferMode(mode_str)
+    except ValueError:
+        raise ValueError(
+            f"Invalid prefer_mode '{mode_str}'. "
+            f"Valid values: default, list_only, regex_only"
+        )