additory 0.1.0a4__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/core/registry.py

@@ -1,177 +0,0 @@
-# registry.py
-# Versioned registry for additory
-
-from dataclasses import dataclass
-from typing import Optional
-import os
-import json
-
-from .logging import log_info, log_warning
-from .config import (
-    get_user_formula_root_override,
-    get_custom_formula_path,
-    get_default_version,
-    get_user_version_override,
-)
-
-from additory.core.loader import load_expression
-from additory.core.parser import parse_expression
-
-
-# ------------------------------------------------------------
-# Resolved Formula Object
-# ------------------------------------------------------------
-
-@dataclass
-class ResolvedFormula:
-    source: str
-    version: str
-    mode: str = "local"
-    ast: Optional[dict] = None
-    sample_clean: Optional[dict] = None
-    sample_unclean: Optional[dict] = None
-
-
-# ------------------------------------------------------------
-# Manifest loading
-# ------------------------------------------------------------
-
-def _load_manifest(root: str, version: str):
-    manifest_path = os.path.join(root, version, "manifest.json")
-
-    if not os.path.exists(manifest_path):
-        raise FileNotFoundError(f"Manifest not found for version {version}")
-
-    with open(manifest_path, "r", encoding="utf-8") as f:
-        return json.load(f)
-
-
-# ------------------------------------------------------------
-# Resolve formula filename
-# ------------------------------------------------------------
-
-def _resolve_filename(formula_name: str, manifest: dict):
-    if formula_name not in manifest:
-        raise FileNotFoundError(f"Formula '{formula_name}' not found in manifest")
-
-    return manifest[formula_name]
-
-
-# ------------------------------------------------------------
-# Main resolver (attaches AST + sample data)
-# ------------------------------------------------------------
-
-def resolve_formula(formula_name: str, namespace="builtin", version=None):
-    """
-    Versioned resolver:
-    1. Custom override path
-    2. User-set formula root
-    3. Version override
-    4. Default version
-    5. Manifest lookup
-    6. Load + parse expression
-    7. Attach AST + sample data
-    
-    Args:
-        formula_name: Name of the formula to resolve
-        namespace: "builtin" or "user" (default: "builtin")
-        version: Specific version to use (optional)
-        
-    Returns:
-        ResolvedFormula object with AST and sample data
-    """
-
-    # --------------------------------------------------------
-    # 1. Custom override (FIXED)
-    # --------------------------------------------------------
-    custom = get_custom_formula_path()
-    if custom:
-        log_info(f"[registry] Using custom formula path: {custom}")
-
-        resolved = ResolvedFormula(source=custom, version="custom")
-
-        text = load_expression(resolved, namespace)
-        parsed = parse_expression(text)
-
-        resolved.ast = parsed.ast
-        resolved.sample_clean = parsed.sample_clean
-        resolved.sample_unclean = parsed.sample_unclean
-
-        return resolved
-
-    # --------------------------------------------------------
-    # 2. Root folder
-    # --------------------------------------------------------
-    root = get_user_formula_root_override()
-    if not root:
-        raise ValueError("Formula root not set. Use add.set_formula_root(path).")
-
-    # --------------------------------------------------------
-    # 3. Version override
-    # --------------------------------------------------------
-    version = (
-        version
-        or get_user_version_override()
-        or get_default_version()
-    )
-
-    # --------------------------------------------------------
-    # 4. Load manifest
-    # --------------------------------------------------------
-    manifest = _load_manifest(root, version)
-
-    # --------------------------------------------------------
-    # 5. Resolve filename
-    # --------------------------------------------------------
-    filename = _resolve_filename(formula_name, manifest)
-
-    # --------------------------------------------------------
-    # 6. Build full path
-    # --------------------------------------------------------
-    full_path = os.path.join(root, version, filename)
-
-    if not os.path.exists(full_path):
-        raise FileNotFoundError(f"Expression file not found: {full_path}")
-
-    resolved = ResolvedFormula(source=full_path, version=version)
-
-    # --------------------------------------------------------
-    # 7. Load + parse + attach AST + sample data
-    # --------------------------------------------------------
-    try:
-        text = load_expression(resolved, namespace)
-        parsed = parse_expression(text)
-
-        resolved.ast = parsed.ast
-        resolved.sample_clean = parsed.sample_clean
-        resolved.sample_unclean = parsed.sample_unclean
-
-        if resolved.ast is None:
-            log_warning(f"[registry] No AST parsed for '{formula_name}'")
-
-    except Exception as e:
-        log_warning(f"[registry] Failed to load/parse '{formula_name}': {e}")
-
-    return resolved
-
-
-# ------------------------------------------------------------
-# Public setters
-# ------------------------------------------------------------
-
-def set_formula_root(path: str):
-    from .config import set_user_formula_root_override
-    set_user_formula_root_override(path)
-    log_info(f"[registry] Formula root set to: {path}")
-
-
-def set_formula_version(v: str):
-    from .config import set_user_version_override
-    set_user_version_override(v)
-    log_info(f"[registry] Version override set to: {v}")
-
-
-def set_custom_formula_path(path: str):
-    from .config import set_custom_formula_path
-    set_custom_formula_path(path)
-    log_info(f"[registry] Custom formula path set to: {path}")

additory/core/sample_data_manager.py

@@ -1,492 +0,0 @@
-# sample_data_manager.py
-# Enhanced sample data management for additory expressions
-
-import os
-import yaml
-import pandas as pd
-from typing import Dict, List, Optional, Any, Union, Tuple
-from dataclasses import dataclass
-import re
-
-from .logging import log_info, log_warning
-from .enhanced_version_manager import EnhancedVersionManager
-from .namespace_manager import NamespaceManager
-from .integrity_manager import IntegrityManager
-
-
-@dataclass
-class SampleDataInfo:
-    """Information about sample data"""
-    expression_name: str
-    version: str
-    has_clean: bool
-    has_unclean: bool
-    clean_rows: int
-    unclean_rows: int
-    educational_comments: List[str]
-    validation_errors: List[str]
-
-
-class SampleDataError(Exception):
-    """Raised when sample data operations fail"""
-    pass
-
-
-class SampleDataManager:
-    """
-    Enhanced sample data management system
-    Provides clean/unclean sample support with educational comments and validation
-    """
-    
-    def __init__(self):
-        self.version_manager = EnhancedVersionManager()
-        self.namespace_manager = NamespaceManager()
-        self.integrity_manager = IntegrityManager()
-        
-        # Sample data validation rules
-        self.validation_rules = {
-            "max_rows": 100,  # Maximum rows in sample data
-            "min_rows": 1,    # Minimum rows in sample data
-            "required_columns": [],  # Will be determined from expression
-            "max_column_length": 50,  # Maximum string length in columns
-        }
-        
-        # Educational comment patterns for unclean data
-        self.educational_patterns = {
-            "missing_values": "# Missing values to test null handling",
-            "invalid_types": "# Invalid data types to test type validation", 
-            "edge_cases": "# Edge cases to test boundary conditions",
-            "malformed_data": "# Malformed data to test error handling",
-            "duplicate_values": "# Duplicate values to test deduplication",
-            "extreme_values": "# Extreme values to test range validation"
-        }
-        
-        log_info("[sample_data] Sample Data Manager initialized")
-    
-    def get_clean_sample(self, expression_name: str, namespace: str = "builtin", 
-                        version: Optional[str] = None) -> pd.DataFrame:
-        """
-        Get clean sample data for an expression
-        
-        Args:
-            expression_name: Name of the expression
-            namespace: Namespace ("builtin" or "user")
-            version: Specific version (optional)
-            
-        Returns:
-            DataFrame with clean sample data
-            
-        Raises:
-            SampleDataError: If sample data cannot be retrieved
-        """
-        try:
-            sample_data = self._get_sample_data(expression_name, namespace, version, "clean")
-            
-            if sample_data is None:
-                # Generate default clean sample if none exists
-                return self._generate_default_clean_sample(expression_name)
-            
-            df = pd.DataFrame(sample_data)
-            
-            # Validate clean sample data
-            validation_errors = self._validate_clean_sample(df, expression_name)
-            if validation_errors:
-                log_warning(f"[sample_data] Clean sample validation issues for {expression_name}: {validation_errors}")
-            
-            log_info(f"[sample_data] Retrieved clean sample for {expression_name} ({len(df)} rows)")
-            return df
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to get clean sample for {expression_name}: {e}")
-            raise SampleDataError(f"Failed to get clean sample data: {e}")
-    
-    def get_unclean_sample(self, expression_name: str, namespace: str = "builtin", 
-                          version: Optional[str] = None) -> pd.DataFrame:
-        """
-        Get unclean sample data with educational comments
-        
-        Args:
-            expression_name: Name of the expression
-            namespace: Namespace ("builtin" or "user")
-            version: Specific version (optional)
-            
-        Returns:
-            DataFrame with unclean sample data and educational comments
-            
-        Raises:
-            SampleDataError: If sample data cannot be retrieved
-        """
-        try:
-            sample_data = self._get_sample_data(expression_name, namespace, version, "unclean")
-            
-            if sample_data is None:
-                # Generate default unclean sample if none exists
-                return self._generate_default_unclean_sample(expression_name)
-            
-            df = pd.DataFrame(sample_data)
-            
-            # Add educational comments as metadata
-            df = self._add_educational_comments(df, expression_name)
-            
-            log_info(f"[sample_data] Retrieved unclean sample for {expression_name} ({len(df)} rows)")
-            return df
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to get unclean sample for {expression_name}: {e}")
-            raise SampleDataError(f"Failed to get unclean sample data: {e}")
-    
-    def validate_sample_data(self, sample_data: Dict[str, Any], 
-                           expression_name: str, sample_type: str = "clean") -> Tuple[bool, List[str]]:
-        """
-        Validate sample data format and content
-        
-        Args:
-            sample_data: Sample data dictionary
-            expression_name: Name of the expression
-            sample_type: "clean" or "unclean"
-            
-        Returns:
-            Tuple of (is_valid, list_of_errors)
-        """
-        errors = []
-        
-        try:
-            # Check if sample_data is a dictionary
-            if not isinstance(sample_data, dict):
-                errors.append("Sample data must be a dictionary")
-                return False, errors
-            
-            # Check if sample_data has columns
-            if not sample_data:
-                errors.append("Sample data cannot be empty")
-                return False, errors
-            
-            # Convert to DataFrame for validation
-            try:
-                df = pd.DataFrame(sample_data)
-            except Exception as e:
-                errors.append(f"Cannot convert sample data to DataFrame: {e}")
-                return False, errors
-            
-            # Validate row count
-            row_count = len(df)
-            if row_count < self.validation_rules["min_rows"]:
-                errors.append(f"Sample data has too few rows: {row_count} < {self.validation_rules['min_rows']}")
-            
-            if row_count > self.validation_rules["max_rows"]:
-                errors.append(f"Sample data has too many rows: {row_count} > {self.validation_rules['max_rows']}")
-            
-            # Validate column content
-            for column, values in sample_data.items():
-                if not isinstance(values, list):
-                    errors.append(f"Column '{column}' must be a list")
-                    continue
-                
-                # Check for consistent length
-                if len(values) != row_count:
-                    errors.append(f"Column '{column}' has inconsistent length")
-                
-                # Check string length limits
-                for i, value in enumerate(values):
-                    if isinstance(value, str) and len(value) > self.validation_rules["max_column_length"]:
-                        errors.append(f"Column '{column}' row {i} exceeds max length")
-            
-            # Specific validation for clean vs unclean samples
-            if sample_type == "clean":
-                errors.extend(self._validate_clean_sample(df, expression_name))
-            else:
-                errors.extend(self._validate_unclean_sample(df, expression_name))
-            
-            is_valid = len(errors) == 0
-            
-            if is_valid:
-                log_info(f"[sample_data] Sample data validation passed for {expression_name}")
-            else:
-                log_warning(f"[sample_data] Sample data validation failed for {expression_name}: {errors}")
-            
-            return is_valid, errors
-            
-        except Exception as e:
-            errors.append(f"Validation error: {e}")
-            return False, errors
-    
-    def get_sample_info(self, expression_name: str, namespace: str = "builtin", 
-                       version: Optional[str] = None) -> SampleDataInfo:
-        """
-        Get comprehensive information about sample data
-        
-        Args:
-            expression_name: Name of the expression
-            namespace: Namespace ("builtin" or "user")
-            version: Specific version (optional)
-            
-        Returns:
-            SampleDataInfo object with comprehensive information
-        """
-        try:
-            # Get sample data
-            clean_data = self._get_sample_data(expression_name, namespace, version, "clean")
-            unclean_data = self._get_sample_data(expression_name, namespace, version, "unclean")
-            
-            # Analyze clean sample
-            has_clean = clean_data is not None
-            clean_rows = len(pd.DataFrame(clean_data)) if has_clean else 0
-            
-            # Analyze unclean sample
-            has_unclean = unclean_data is not None
-            unclean_rows = len(pd.DataFrame(unclean_data)) if has_unclean else 0
-            
-            # Extract educational comments
-            educational_comments = []
-            if has_unclean:
-                educational_comments = self._extract_educational_comments(unclean_data)
-            
-            # Validate samples
-            validation_errors = []
-            if has_clean:
-                _, clean_errors = self.validate_sample_data(clean_data, expression_name, "clean")
-                validation_errors.extend([f"Clean: {err}" for err in clean_errors])
-            
-            if has_unclean:
-                _, unclean_errors = self.validate_sample_data(unclean_data, expression_name, "unclean")
-                validation_errors.extend([f"Unclean: {err}" for err in unclean_errors])
-            
-            return SampleDataInfo(
-                expression_name=expression_name,
-                version=version or self.version_manager.default_version,
-                has_clean=has_clean,
-                has_unclean=has_unclean,
-                clean_rows=clean_rows,
-                unclean_rows=unclean_rows,
-                educational_comments=educational_comments,
-                validation_errors=validation_errors
-            )
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to get sample info for {expression_name}: {e}")
-            return SampleDataInfo(
-                expression_name=expression_name,
-                version=version or "unknown",
-                has_clean=False,
-                has_unclean=False,
-                clean_rows=0,
-                unclean_rows=0,
-                educational_comments=[],
-                validation_errors=[f"Failed to get sample info: {e}"]
-            )
-    
-    def create_sample_template(self, expression_name: str, columns: List[str]) -> Dict[str, Dict[str, Any]]:
-        """
-        Create a template for sample data
-        
-        Args:
-            expression_name: Name of the expression
-            columns: List of required columns
-            
-        Returns:
-            Dictionary with clean and unclean sample templates
-        """
-        try:
-            # Create clean sample template
-            clean_template = {}
-            for column in columns:
-                clean_template[column] = [f"sample_{column}_1", f"sample_{column}_2", f"sample_{column}_3"]
-            
-            # Create unclean sample template with educational comments
-            unclean_template = {}
-            for column in columns:
-                unclean_template[column] = [
-                    f"valid_{column}",
-                    None,  # Missing value
-                    f"invalid_{column}_type",
-                    f"extreme_{column}_value"
-                ]
-            
-            # Add educational comments
-            unclean_template["_comments"] = [
-                "# This is unclean sample data for testing error handling",
-                "# Row 1: Valid data",
-                "# Row 2: Missing values (None/null)",
-                "# Row 3: Invalid data types",
-                "# Row 4: Extreme or edge case values"
-            ]
-            
-            template = {
-                "clean": clean_template,
-                "unclean": unclean_template
-            }
-            
-            log_info(f"[sample_data] Created sample template for {expression_name}")
-            return template
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to create sample template for {expression_name}: {e}")
-            raise SampleDataError(f"Failed to create sample template: {e}")
-    
-    def _get_sample_data(self, expression_name: str, namespace: str, 
-                        version: Optional[str], sample_type: str) -> Optional[Dict[str, Any]]:
-        """Get raw sample data from expression file"""
-        try:
-            # Get expression file path
-            expression_path = self.namespace_manager.get_expression_file_path(
-                namespace, expression_name, version
-            )
-            
-            if not expression_path or not os.path.exists(expression_path):
-                return None
-            
-            # Validate integrity
-            if not self.integrity_manager.validate_integrity(expression_path):
-                log_warning(f"[sample_data] Integrity validation failed for {expression_path}")
-                return None
-            
-            # Parse expression file
-            with open(expression_path, 'r') as f:
-                content = yaml.safe_load(f)
-            
-            # Extract sample data
-            sample_section = content.get("sample", {})
-            return sample_section.get(sample_type)
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to get sample data from {expression_path}: {e}")
-            return None
-    
-    def _validate_clean_sample(self, df: pd.DataFrame, expression_name: str) -> List[str]:
-        """Validate clean sample data"""
-        errors = []
-        
-        # Check for missing values in clean sample
-        if df.isnull().any().any():
-            errors.append("Clean sample should not contain missing values")
-        
-        # Check for reasonable data types
-        for column in df.columns:
-            if column.startswith('_'):  # Skip metadata columns
-                continue
-                
-            series = df[column]
-            
-            # Check for mixed types (should be consistent in clean data)
-            unique_types = set(type(x).__name__ for x in series.dropna())
-            if len(unique_types) > 1:
-                errors.append(f"Column '{column}' has mixed data types in clean sample")
-        
-        return errors
-    
-    def _validate_unclean_sample(self, df: pd.DataFrame, expression_name: str) -> List[str]:
-        """Validate unclean sample data"""
-        errors = []
-        
-        # Unclean samples should have some issues for educational purposes
-        has_nulls = df.isnull().any().any()
-        has_mixed_types = False
-        
-        for column in df.columns:
-            if column.startswith('_'):  # Skip metadata columns
-                continue
-                
-            series = df[column]
-            unique_types = set(type(x).__name__ for x in series.dropna())
-            if len(unique_types) > 1:
-                has_mixed_types = True
-                break
-        
-        # Unclean samples should demonstrate common data issues
-        if not has_nulls and not has_mixed_types:
-            errors.append("Unclean sample should contain some data quality issues for educational purposes")
-        
-        return errors
-    
-    def _add_educational_comments(self, df: pd.DataFrame, expression_name: str) -> pd.DataFrame:
-        """Add educational comments to unclean sample data"""
-        try:
-            # Add a comments column with educational information
-            comments = []
-            
-            for i, row in df.iterrows():
-                comment_parts = []
-                
-                # Check for missing values
-                if row.isnull().any():
-                    comment_parts.append("Contains missing values")
-                
-                # Check for potential type issues
-                for col, val in row.items():
-                    if col.startswith('_'):
-                        continue
-                    if isinstance(val, str) and val.lower() in ['invalid', 'error', 'null']:
-                        comment_parts.append(f"'{col}' has invalid value")
-                
-                if not comment_parts:
-                    comment_parts.append("Valid data row")
-                
-                comments.append(" | ".join(comment_parts))
-            
-            # Add comments as a new column
-            df_with_comments = df.copy()
-            df_with_comments['_educational_comments'] = comments
-            
-            return df_with_comments
-            
-        except Exception as e:
-            log_warning(f"[sample_data] Failed to add educational comments: {e}")
-            return df
-    
-    def _extract_educational_comments(self, sample_data: Dict[str, Any]) -> List[str]:
-        """Extract educational comments from sample data"""
-        comments = []
-        
-        # Look for comment fields
-        if '_comments' in sample_data:
-            comments.extend(sample_data['_comments'])
-        
-        # Generate comments based on data patterns
-        try:
-            df = pd.DataFrame({k: v for k, v in sample_data.items() if not k.startswith('_')})
-            
-            if df.isnull().any().any():
-                comments.append("Contains missing values for null handling testing")
-            
-            for column in df.columns:
-                series = df[column]
-                unique_types = set(type(x).__name__ for x in series.dropna())
-                if len(unique_types) > 1:
-                    comments.append(f"Column '{column}' has mixed types for type validation testing")
-        
-        except Exception:
-            pass  # Ignore errors in comment extraction
-        
-        return comments
-    
-    def _generate_default_clean_sample(self, expression_name: str) -> pd.DataFrame:
-        """Generate default clean sample data when none exists"""
-        return pd.DataFrame({
-            "col_a": [1, 2, 3],
-            "col_b": [4, 5, 6],
-            "_info": [f"Default clean sample for '{expression_name}'"] * 3
-        })
-    
-    def _generate_default_unclean_sample(self, expression_name: str) -> pd.DataFrame:
-        """Generate default unclean sample data when none exists"""
-        return pd.DataFrame({
-            "col_a": [1, None, "invalid"],
-            "col_b": [4, 5, -999],
-            "_educational_comments": [
-                "Valid data row",
-                "Missing value in col_a",
-                "Invalid type in col_a, extreme value in col_b"
-            ],
-            "_info": [f"Default unclean sample for '{expression_name}'"] * 3
-        })
-
-
-# Global sample data manager instance
-_sample_data_manager = None
-
-def get_sample_data_manager() -> SampleDataManager:
-    """Get the global sample data manager instance"""
-    global _sample_data_manager
-    if _sample_data_manager is None:
-        _sample_data_manager = SampleDataManager()
-    return _sample_data_manager