additory 0.1.0a4__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/expressions/samples.py

@@ -1,492 +0,0 @@
-# sample_data_manager.py
-# Enhanced sample data management for additory expressions
-
-import os
-import yaml
-import pandas as pd
-from typing import Dict, List, Optional, Any, Union, Tuple
-from dataclasses import dataclass
-import re
-
-from .logging import log_info, log_warning
-from .enhanced_version_manager import EnhancedVersionManager
-from .namespace_manager import NamespaceManager
-from .integrity_manager import IntegrityManager
-
-
-@dataclass
-class SampleDataInfo:
-    """Information about sample data"""
-    expression_name: str
-    version: str
-    has_clean: bool
-    has_unclean: bool
-    clean_rows: int
-    unclean_rows: int
-    educational_comments: List[str]
-    validation_errors: List[str]
-
-
-class SampleDataError(Exception):
-    """Raised when sample data operations fail"""
-    pass
-
-
-class SampleDataManager:
-    """
-    Enhanced sample data management system
-    Provides clean/unclean sample support with educational comments and validation
-    """
-    
-    def __init__(self):
-        self.version_manager = EnhancedVersionManager()
-        self.namespace_manager = NamespaceManager()
-        self.integrity_manager = IntegrityManager()
-        
-        # Sample data validation rules
-        self.validation_rules = {
-            "max_rows": 100,  # Maximum rows in sample data
-            "min_rows": 1,    # Minimum rows in sample data
-            "required_columns": [],  # Will be determined from expression
-            "max_column_length": 50,  # Maximum string length in columns
-        }
-        
-        # Educational comment patterns for unclean data
-        self.educational_patterns = {
-            "missing_values": "# Missing values to test null handling",
-            "invalid_types": "# Invalid data types to test type validation", 
-            "edge_cases": "# Edge cases to test boundary conditions",
-            "malformed_data": "# Malformed data to test error handling",
-            "duplicate_values": "# Duplicate values to test deduplication",
-            "extreme_values": "# Extreme values to test range validation"
-        }
-        
-        log_info("[sample_data] Sample Data Manager initialized")
-    
-    def get_clean_sample(self, expression_name: str, namespace: str = "builtin", 
-                        version: Optional[str] = None) -> pd.DataFrame:
-        """
-        Get clean sample data for an expression
-        
-        Args:
-            expression_name: Name of the expression
-            namespace: Namespace ("builtin" or "user")
-            version: Specific version (optional)
-            
-        Returns:
-            DataFrame with clean sample data
-            
-        Raises:
-            SampleDataError: If sample data cannot be retrieved
-        """
-        try:
-            sample_data = self._get_sample_data(expression_name, namespace, version, "clean")
-            
-            if sample_data is None:
-                # Generate default clean sample if none exists
-                return self._generate_default_clean_sample(expression_name)
-            
-            df = pd.DataFrame(sample_data)
-            
-            # Validate clean sample data
-            validation_errors = self._validate_clean_sample(df, expression_name)
-            if validation_errors:
-                log_warning(f"[sample_data] Clean sample validation issues for {expression_name}: {validation_errors}")
-            
-            log_info(f"[sample_data] Retrieved clean sample for {expression_name} ({len(df)} rows)")
-            return df
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to get clean sample for {expression_name}: {e}")
-            raise SampleDataError(f"Failed to get clean sample data: {e}")
-    
-    def get_unclean_sample(self, expression_name: str, namespace: str = "builtin", 
-                          version: Optional[str] = None) -> pd.DataFrame:
-        """
-        Get unclean sample data with educational comments
-        
-        Args:
-            expression_name: Name of the expression
-            namespace: Namespace ("builtin" or "user")
-            version: Specific version (optional)
-            
-        Returns:
-            DataFrame with unclean sample data and educational comments
-            
-        Raises:
-            SampleDataError: If sample data cannot be retrieved
-        """
-        try:
-            sample_data = self._get_sample_data(expression_name, namespace, version, "unclean")
-            
-            if sample_data is None:
-                # Generate default unclean sample if none exists
-                return self._generate_default_unclean_sample(expression_name)
-            
-            df = pd.DataFrame(sample_data)
-            
-            # Add educational comments as metadata
-            df = self._add_educational_comments(df, expression_name)
-            
-            log_info(f"[sample_data] Retrieved unclean sample for {expression_name} ({len(df)} rows)")
-            return df
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to get unclean sample for {expression_name}: {e}")
-            raise SampleDataError(f"Failed to get unclean sample data: {e}")
-    
-    def validate_sample_data(self, sample_data: Dict[str, Any], 
-                           expression_name: str, sample_type: str = "clean") -> Tuple[bool, List[str]]:
-        """
-        Validate sample data format and content
-        
-        Args:
-            sample_data: Sample data dictionary
-            expression_name: Name of the expression
-            sample_type: "clean" or "unclean"
-            
-        Returns:
-            Tuple of (is_valid, list_of_errors)
-        """
-        errors = []
-        
-        try:
-            # Check if sample_data is a dictionary
-            if not isinstance(sample_data, dict):
-                errors.append("Sample data must be a dictionary")
-                return False, errors
-            
-            # Check if sample_data has columns
-            if not sample_data:
-                errors.append("Sample data cannot be empty")
-                return False, errors
-            
-            # Convert to DataFrame for validation
-            try:
-                df = pd.DataFrame(sample_data)
-            except Exception as e:
-                errors.append(f"Cannot convert sample data to DataFrame: {e}")
-                return False, errors
-            
-            # Validate row count
-            row_count = len(df)
-            if row_count < self.validation_rules["min_rows"]:
-                errors.append(f"Sample data has too few rows: {row_count} < {self.validation_rules['min_rows']}")
-            
-            if row_count > self.validation_rules["max_rows"]:
-                errors.append(f"Sample data has too many rows: {row_count} > {self.validation_rules['max_rows']}")
-            
-            # Validate column content
-            for column, values in sample_data.items():
-                if not isinstance(values, list):
-                    errors.append(f"Column '{column}' must be a list")
-                    continue
-                
-                # Check for consistent length
-                if len(values) != row_count:
-                    errors.append(f"Column '{column}' has inconsistent length")
-                
-                # Check string length limits
-                for i, value in enumerate(values):
-                    if isinstance(value, str) and len(value) > self.validation_rules["max_column_length"]:
-                        errors.append(f"Column '{column}' row {i} exceeds max length")
-            
-            # Specific validation for clean vs unclean samples
-            if sample_type == "clean":
-                errors.extend(self._validate_clean_sample(df, expression_name))
-            else:
-                errors.extend(self._validate_unclean_sample(df, expression_name))
-            
-            is_valid = len(errors) == 0
-            
-            if is_valid:
-                log_info(f"[sample_data] Sample data validation passed for {expression_name}")
-            else:
-                log_warning(f"[sample_data] Sample data validation failed for {expression_name}: {errors}")
-            
-            return is_valid, errors
-            
-        except Exception as e:
-            errors.append(f"Validation error: {e}")
-            return False, errors
-    
-    def get_sample_info(self, expression_name: str, namespace: str = "builtin", 
-                       version: Optional[str] = None) -> SampleDataInfo:
-        """
-        Get comprehensive information about sample data
-        
-        Args:
-            expression_name: Name of the expression
-            namespace: Namespace ("builtin" or "user")
-            version: Specific version (optional)
-            
-        Returns:
-            SampleDataInfo object with comprehensive information
-        """
-        try:
-            # Get sample data
-            clean_data = self._get_sample_data(expression_name, namespace, version, "clean")
-            unclean_data = self._get_sample_data(expression_name, namespace, version, "unclean")
-            
-            # Analyze clean sample
-            has_clean = clean_data is not None
-            clean_rows = len(pd.DataFrame(clean_data)) if has_clean else 0
-            
-            # Analyze unclean sample
-            has_unclean = unclean_data is not None
-            unclean_rows = len(pd.DataFrame(unclean_data)) if has_unclean else 0
-            
-            # Extract educational comments
-            educational_comments = []
-            if has_unclean:
-                educational_comments = self._extract_educational_comments(unclean_data)
-            
-            # Validate samples
-            validation_errors = []
-            if has_clean:
-                _, clean_errors = self.validate_sample_data(clean_data, expression_name, "clean")
-                validation_errors.extend([f"Clean: {err}" for err in clean_errors])
-            
-            if has_unclean:
-                _, unclean_errors = self.validate_sample_data(unclean_data, expression_name, "unclean")
-                validation_errors.extend([f"Unclean: {err}" for err in unclean_errors])
-            
-            return SampleDataInfo(
-                expression_name=expression_name,
-                version=version or self.version_manager.default_version,
-                has_clean=has_clean,
-                has_unclean=has_unclean,
-                clean_rows=clean_rows,
-                unclean_rows=unclean_rows,
-                educational_comments=educational_comments,
-                validation_errors=validation_errors
-            )
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to get sample info for {expression_name}: {e}")
-            return SampleDataInfo(
-                expression_name=expression_name,
-                version=version or "unknown",
-                has_clean=False,
-                has_unclean=False,
-                clean_rows=0,
-                unclean_rows=0,
-                educational_comments=[],
-                validation_errors=[f"Failed to get sample info: {e}"]
-            )
-    
-    def create_sample_template(self, expression_name: str, columns: List[str]) -> Dict[str, Dict[str, Any]]:
-        """
-        Create a template for sample data
-        
-        Args:
-            expression_name: Name of the expression
-            columns: List of required columns
-            
-        Returns:
-            Dictionary with clean and unclean sample templates
-        """
-        try:
-            # Create clean sample template
-            clean_template = {}
-            for column in columns:
-                clean_template[column] = [f"sample_{column}_1", f"sample_{column}_2", f"sample_{column}_3"]
-            
-            # Create unclean sample template with educational comments
-            unclean_template = {}
-            for column in columns:
-                unclean_template[column] = [
-                    f"valid_{column}",
-                    None,  # Missing value
-                    f"invalid_{column}_type",
-                    f"extreme_{column}_value"
-                ]
-            
-            # Add educational comments
-            unclean_template["_comments"] = [
-                "# This is unclean sample data for testing error handling",
-                "# Row 1: Valid data",
-                "# Row 2: Missing values (None/null)",
-                "# Row 3: Invalid data types",
-                "# Row 4: Extreme or edge case values"
-            ]
-            
-            template = {
-                "clean": clean_template,
-                "unclean": unclean_template
-            }
-            
-            log_info(f"[sample_data] Created sample template for {expression_name}")
-            return template
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to create sample template for {expression_name}: {e}")
-            raise SampleDataError(f"Failed to create sample template: {e}")
-    
-    def _get_sample_data(self, expression_name: str, namespace: str, 
-                        version: Optional[str], sample_type: str) -> Optional[Dict[str, Any]]:
-        """Get raw sample data from expression file"""
-        try:
-            # Get expression file path
-            expression_path = self.namespace_manager.get_expression_file_path(
-                namespace, expression_name, version
-            )
-            
-            if not expression_path or not os.path.exists(expression_path):
-                return None
-            
-            # Validate integrity
-            if not self.integrity_manager.validate_integrity(expression_path):
-                log_warning(f"[sample_data] Integrity validation failed for {expression_path}")
-                return None
-            
-            # Parse expression file
-            with open(expression_path, 'r') as f:
-                content = yaml.safe_load(f)
-            
-            # Extract sample data
-            sample_section = content.get("sample", {})
-            return sample_section.get(sample_type)
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to get sample data from {expression_path}: {e}")
-            return None
-    
-    def _validate_clean_sample(self, df: pd.DataFrame, expression_name: str) -> List[str]:
-        """Validate clean sample data"""
-        errors = []
-        
-        # Check for missing values in clean sample
-        if df.isnull().any().any():
-            errors.append("Clean sample should not contain missing values")
-        
-        # Check for reasonable data types
-        for column in df.columns:
-            if column.startswith('_'):  # Skip metadata columns
-                continue
-                
-            series = df[column]
-            
-            # Check for mixed types (should be consistent in clean data)
-            unique_types = set(type(x).__name__ for x in series.dropna())
-            if len(unique_types) > 1:
-                errors.append(f"Column '{column}' has mixed data types in clean sample")
-        
-        return errors
-    
-    def _validate_unclean_sample(self, df: pd.DataFrame, expression_name: str) -> List[str]:
-        """Validate unclean sample data"""
-        errors = []
-        
-        # Unclean samples should have some issues for educational purposes
-        has_nulls = df.isnull().any().any()
-        has_mixed_types = False
-        
-        for column in df.columns:
-            if column.startswith('_'):  # Skip metadata columns
-                continue
-                
-            series = df[column]
-            unique_types = set(type(x).__name__ for x in series.dropna())
-            if len(unique_types) > 1:
-                has_mixed_types = True
-                break
-        
-        # Unclean samples should demonstrate common data issues
-        if not has_nulls and not has_mixed_types:
-            errors.append("Unclean sample should contain some data quality issues for educational purposes")
-        
-        return errors
-    
-    def _add_educational_comments(self, df: pd.DataFrame, expression_name: str) -> pd.DataFrame:
-        """Add educational comments to unclean sample data"""
-        try:
-            # Add a comments column with educational information
-            comments = []
-            
-            for i, row in df.iterrows():
-                comment_parts = []
-                
-                # Check for missing values
-                if row.isnull().any():
-                    comment_parts.append("Contains missing values")
-                
-                # Check for potential type issues
-                for col, val in row.items():
-                    if col.startswith('_'):
-                        continue
-                    if isinstance(val, str) and val.lower() in ['invalid', 'error', 'null']:
-                        comment_parts.append(f"'{col}' has invalid value")
-                
-                if not comment_parts:
-                    comment_parts.append("Valid data row")
-                
-                comments.append(" | ".join(comment_parts))
-            
-            # Add comments as a new column
-            df_with_comments = df.copy()
-            df_with_comments['_educational_comments'] = comments
-            
-            return df_with_comments
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to add educational comments: {e}")
-            return df
-    
-    def _extract_educational_comments(self, sample_data: Dict[str, Any]) -> List[str]:
-        """Extract educational comments from sample data"""
-        comments = []
-        
-        # Look for comment fields
-        if '_comments' in sample_data:
-            comments.extend(sample_data['_comments'])
-        
-        # Generate comments based on data patterns
-        try:
-            df = pd.DataFrame({k: v for k, v in sample_data.items() if not k.startswith('_')})
-            
-            if df.isnull().any().any():
-                comments.append("Contains missing values for null handling testing")
-            
-            for column in df.columns:
-                series = df[column]
-                unique_types = set(type(x).__name__ for x in series.dropna())
-                if len(unique_types) > 1:
-                    comments.append(f"Column '{column}' has mixed types for type validation testing")
-        
-        except Exception:
-            pass  # Ignore errors in comment extraction
-        
-        return comments
-    
-    def _generate_default_clean_sample(self, expression_name: str) -> pd.DataFrame:
-        """Generate default clean sample data when none exists"""
-        return pd.DataFrame({
-            "col_a": [1, 2, 3],
-            "col_b": [4, 5, 6],
-            "_info": [f"Default clean sample for '{expression_name}'"] * 3
-        })
-    
-    def _generate_default_unclean_sample(self, expression_name: str) -> pd.DataFrame:
-        """Generate default unclean sample data when none exists"""
-        return pd.DataFrame({
-            "col_a": [1, None, "invalid"],
-            "col_b": [4, 5, -999],
-            "_educational_comments": [
-                "Valid data row",
-                "Missing value in col_a",
-                "Invalid type in col_a, extreme value in col_b"
-            ],
-            "_info": [f"Default unclean sample for '{expression_name}'"] * 3
-        })
-
-
-# Global sample data manager instance
-_sample_data_manager = None
-
-def get_sample_data_manager() -> SampleDataManager:
-    """Get the global sample data manager instance"""
-    global _sample_data_manager
-    if _sample_data_manager is None:
-        _sample_data_manager = SampleDataManager()
-    return _sample_data_manager

additory/synthetic/__init__.py

@@ -1,13 +0,0 @@
-"""
-Synthetic Module - Synthetic Data Generation Functionality
-
-This module provides synthetic data generation capabilities to add synthetic rows
-to existing dataframes or create data from scratch by intelligently sampling 
-from existing data patterns.
-"""
-
-from additory.synthetic.synthesizer import synthetic
-
-__all__ = [
-    "synthetic"
-]

additory/synthetic/column_name_resolver.py

@@ -1,149 +0,0 @@
-"""
-Column Name Resolver for Linked Lists
-
-Resolves column names for linked lists using priority order:
-1. Column_Names row (explicit names)
-2. Underscore parsing from list name
-3. Fallback to {strategy_key}_1, {strategy_key}_2, etc.
-"""
-
-from typing import List, Optional
-import warnings
-
-
-def parse_column_names_from_underscores(list_name: str) -> Optional[List[str]]:
-    """
-    Parse column names from list name using underscore delimiters.
-    
-    Args:
-        list_name: Name of the list variable (e.g., "AE_CM_SEV")
-        
-    Returns:
-        List of column names, or None if no underscores found
-        
-    Examples:
-        >>> parse_column_names_from_underscores("AE_CM_SEV")
-        ['AE', 'CM', 'SEV']
-        
-        >>> parse_column_names_from_underscores("adverse_event_medication")
-        ['adverse', 'event', 'medication']
-        
-        >>> parse_column_names_from_underscores("adverseconmed")
-        None
-    """
-    if '_' not in list_name:
-        return None
-    
-    parts = list_name.split('_')
-    
-    # Filter out empty parts
-    column_names = [part for part in parts if part]
-    
-    if not column_names:
-        return None
-    
-    return column_names
-
-
-def generate_fallback_column_names(strategy_key: str, num_columns: int) -> List[str]:
-    """
-    Generate fallback column names when no other naming strategy works.
-    
-    Format: {strategy_key}_1, {strategy_key}_2, etc.
-    
-    Args:
-        strategy_key: Key from strategy dict (e.g., "col1")
-        num_columns: Number of columns to generate names for
-        
-    Returns:
-        List of column names
-        
-    Examples:
-        >>> generate_fallback_column_names("col1", 3)
-        ['col1_1', 'col1_2', 'col1_3']
-        
-        >>> generate_fallback_column_names("adverse_events", 2)
-        ['adverse_events_1', 'adverse_events_2']
-    """
-    return [f"{strategy_key}_{i+1}" for i in range(num_columns)]
-
-
-def resolve_column_names(
-    list_name: str,
-    strategy_key: str,
-    num_columns: int,
-    explicit_names: Optional[List[str]] = None
-) -> List[str]:
-    """
-    Resolve column names using priority order.
-    
-    Priority:
-    1. explicit_names (from Column_Names row)
-    2. Underscore parsing from list_name
-    3. Fallback to {strategy_key}_1, {strategy_key}_2, etc.
-    
-    Args:
-        list_name: Name of the list variable
-        strategy_key: Key from strategy dict
-        num_columns: Number of columns to generate
-        explicit_names: Explicit column names from Column_Names row (optional)
-        
-    Returns:
-        List of column names
-        
-    Raises:
-        ValueError: If explicit_names count doesn't match num_columns
-        
-    Examples:
-        >>> # Priority 1: Explicit names
-        >>> resolve_column_names("AE_CM", "col1", 2, ["adverse_event", "medication"])
-        ['adverse_event', 'medication']
-        
-        >>> # Priority 2: Underscore parsing
-        >>> resolve_column_names("AE_CM_SEV", "col1", 3)
-        ['AE', 'CM', 'SEV']
-        
-        >>> # Priority 3: Fallback
-        >>> resolve_column_names("adverseconmed", "col1", 2)
-        ['col1_1', 'col1_2']
-    """
-    # Priority 1: Explicit names from Column_Names row
-    if explicit_names is not None:
-        if len(explicit_names) != num_columns:
-            raise ValueError(
-                f"Column_Names row has {len(explicit_names)} names but "
-                f"linked list generates {num_columns} columns. They must match."
-            )
-        return explicit_names
-    
-    # Priority 2: Underscore parsing
-    parsed_names = parse_column_names_from_underscores(list_name)
-    if parsed_names is not None:
-        if len(parsed_names) == num_columns:
-            return parsed_names
-        else:
-            # Underscore count doesn't match - fall through to fallback
-            warnings.warn(
-                f"List name '{list_name}' has {len(parsed_names)} underscore-separated "
-                f"parts but generates {num_columns} columns. "
-                f"Using fallback naming: {strategy_key}_1, {strategy_key}_2, etc.\n"
-                f"Suggestion: Use a list name with {num_columns-1} underscores, "
-                f"or add a Column_Names row for explicit naming.",
-                UserWarning
-            )
-    else:
-        # No underscores - emit warning
-        warnings.warn(
-            f"List name '{list_name}' has no underscores. "
-            f"Using fallback naming: {strategy_key}_1, {strategy_key}_2, etc.\n"
-            f"Suggestion: Use underscore-delimited naming (e.g., 'AE_CM_SEV') "
-            f"or add a Column_Names row:\n"
-            f"  {list_name} = [\n"
-            f"      ['Column_Names:[col1,col2,col3]'],\n"
-            f"      ['primary', ['attr1'], ['attr2']]\n"
-            f"  ]",
-            UserWarning
-        )
-    
-    # Priority 3: Fallback
-    return generate_fallback_column_names(strategy_key, num_columns)