additory 0.1.0a3__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/expressions/resolver.py

@@ -0,0 +1,274 @@
+"""
+Expression dependency resolver for Additory.
+
+Resolves column references and dependencies in expressions.
+"""
+
+import re
+from typing import List, Dict, Set
+
+
+# Known function names
+FUNCTION_NAMES = {
+    # Math functions
+    'sqrt', 'abs', 'log', 'log10', 'exp', 'pow', 'round', 'floor', 'ceil',
+    # String functions
+    'lower', 'upper', 'trim', 'length', 'substring', 'replace', 'contains', 'matches',
+    # DateTime functions
+    'year', 'month', 'day', 'hour', 'minute', 'second', 'day_of_week', 'time_of_day',
+    # Aggregation functions
+    'sum', 'mean', 'median', 'min', 'max', 'count', 'std',
+    # Conditional functions
+    'if_else', 'coalesce', 'is_null', 'is_not_null'
+}
+
+
+def resolve_dependencies(expression: str, available_columns: List[str]) -> List[str]:
+    """
+    Find all column dependencies in an expression.
+    
+    Args:
+        expression: Expression string
+        available_columns: List of available column names in DataFrame
+        
+    Returns:
+        List of column names used in expression
+        
+    Example:
+        deps = resolve_dependencies('weight / (height ** 2)', ['weight', 'height', 'age'])
+        # Returns: ['weight', 'height']
+    """
+    # Extract all identifiers
+    identifiers = extract_identifiers(expression)
+    
+    # Filter to only columns that exist in DataFrame
+    dependencies = [id for id in identifiers if id in available_columns]
+    
+    # Return unique list
+    return list(set(dependencies))
+
+
+def validate_dependencies(expression: str, available_columns: List[str]) -> bool:
+    """
+    Validate that all column dependencies exist.
+    
+    Args:
+        expression: Expression string
+        available_columns: List of available column names
+        
+    Returns:
+        True if all dependencies exist
+        
+    Raises:
+        ValueError: If any dependencies are missing
+        
+    Example:
+        validate_dependencies('weight / height', ['weight', 'height'])  # Returns True
+        validate_dependencies('weight / height', ['age'])  # Raises ValueError
+    """
+    # Extract all identifiers
+    identifiers = extract_identifiers(expression)
+    
+    # Find missing columns
+    missing = [id for id in identifiers if id not in available_columns]
+    
+    if missing:
+        raise ValueError(
+            f"Expression requires columns {missing} but DataFrame only has {available_columns}"
+        )
+    
+    return True
+
+
+def extract_identifiers(expression: str) -> List[str]:
+    """
+    Extract all identifiers from expression string.
+    
+    Args:
+        expression: Expression string
+        
+    Returns:
+        List of identifier names (column names, not function names)
+        
+    Example:
+        identifiers = extract_identifiers('sqrt((height * weight) / 3600)')
+        # Returns: ['height', 'weight']
+    """
+    # Remove string literals first (both single and double quotes)
+    # This prevents extracting identifiers from within strings
+    expr_without_strings = re.sub(r'"[^"]*"', '', expression)
+    expr_without_strings = re.sub(r"'[^']*'", '', expr_without_strings)
+    
+    # Pattern to match identifiers (letters, numbers, underscores)
+    pattern = r'\b[a-zA-Z_][a-zA-Z0-9_]*\b'
+    
+    # Find all matches
+    matches = re.findall(pattern, expr_without_strings)
+    
+    # Filter out keywords and function names
+    keywords = {'AND', 'OR', 'NOT'}
+    identifiers = [
+        match for match in matches
+        if match not in keywords and not is_function_name(match)
+    ]
+    
+    # Return unique list
+    return list(set(identifiers))
+
+
+def is_function_name(identifier: str) -> bool:
+    """
+    Check if identifier is a function name.
+    
+    Args:
+        identifier: Identifier string
+        
+    Returns:
+        True if function name, False if column name
+        
+    Example:
+        is_function_name('sqrt')  # Returns True
+        is_function_name('weight')  # Returns False
+    """
+    return identifier.lower() in FUNCTION_NAMES
+
+
+def check_circular_dependencies(expressions: Dict[str, str]) -> bool:
+    """
+    Check for circular dependencies between expressions.
+    
+    Args:
+        expressions: Dictionary mapping expression name to expression string
+        
+    Returns:
+        True if no circular dependencies
+        
+    Raises:
+        ValueError: If circular dependency detected
+        
+    Example:
+        expressions = {
+            'a': 'b + 1',
+            'b': 'c + 1',
+            'c': 'a + 1'  # Circular!
+        }
+        check_circular_dependencies(expressions)
+        # Raises: "Circular dependency detected: a -> b -> c -> a"
+    """
+    # Build dependency graph
+    graph = build_dependency_graph(expressions)
+    
+    # Track visited nodes and recursion stack
+    visited: Set[str] = set()
+    rec_stack: Set[str] = set()
+    path: List[str] = []
+    
+    def has_cycle(node: str) -> bool:
+        """DFS to detect cycles."""
+        visited.add(node)
+        rec_stack.add(node)
+        path.append(node)
+        
+        # Check all dependencies
+        for dep in graph.get(node, []):
+            if dep not in visited:
+                if has_cycle(dep):
+                    return True
+            elif dep in rec_stack:
+                # Found cycle
+                cycle_start = path.index(dep)
+                cycle = path[cycle_start:] + [dep]
+                raise ValueError(
+                    f"Circular dependency detected: {' -> '.join(cycle)}"
+                )
+        
+        rec_stack.remove(node)
+        path.pop()
+        return False
+    
+    # Check each expression
+    for expr_name in expressions:
+        if expr_name not in visited:
+            has_cycle(expr_name)
+    
+    return True
+
+
+def topological_sort(expressions: Dict[str, str]) -> List[str]:
+    """
+    Sort expressions by dependency order.
+    
+    Args:
+        expressions: Dictionary mapping expression name to expression string
+        
+    Returns:
+        List of expression names in dependency order
+        
+    Example:
+        expressions = {
+            'bmi': 'weight / (height ** 2)',
+            'bmi_category': 'if_else(bmi < 18.5, "underweight", "normal")',
+            'weight': 'weight_lb * 0.453592'
+        }
+        order = topological_sort(expressions)
+        # Returns: ['weight', 'bmi', 'bmi_category']
+    """
+    # Build dependency graph
+    graph = build_dependency_graph(expressions)
+    
+    # Track visited nodes and result
+    visited: Set[str] = set()
+    result: List[str] = []
+    
+    def visit(node: str):
+        """DFS to build topological order."""
+        if node in visited:
+            return
+        
+        visited.add(node)
+        
+        # Visit all dependencies first
+        for dep in graph.get(node, []):
+            if dep in expressions:  # Only visit if it's an expression
+                visit(dep)
+        
+        # Add node after all dependencies
+        result.append(node)
+    
+    # Visit each expression
+    for expr_name in expressions:
+        visit(expr_name)
+    
+    return result
+
+
+def build_dependency_graph(expressions: Dict[str, str]) -> Dict[str, List[str]]:
+    """
+    Build dependency graph for expressions.
+    
+    Args:
+        expressions: Dictionary mapping expression name to expression string
+        
+    Returns:
+        Dictionary mapping expression name to list of dependencies
+        
+    Example:
+        expressions = {
+            'bmi': 'weight / (height ** 2)',
+            'bmi_category': 'if_else(bmi < 18.5, "underweight", "normal")'
+        }
+        graph = build_dependency_graph(expressions)
+        # Returns: {'bmi': [], 'bmi_category': ['bmi']}
+    """
+    graph: Dict[str, List[str]] = {}
+    
+    for expr_name, expr_string in expressions.items():
+        # Extract identifiers from expression
+        identifiers = extract_identifiers(expr_string)
+        
+        # Filter to only other expressions (not columns)
+        dependencies = [id for id in identifiers if id in expressions]
+        
+        graph[expr_name] = dependencies
+    
+    return graph

additory/functions/__init__.py

@@ -0,0 +1 @@
+"""Functions package."""

additory/functions/analyze/__init__.py

@@ -0,0 +1,144 @@
+"""
+Analyze function - Data analysis with multiple modes.
+
+Provides comprehensive data analysis including quality, cardinality,
+distributions, correlations, and more.
+"""
+
+import polars as pl
+import time
+from typing import Optional
+
+from additory.core.backend import detect_backend, to_polars
+from additory.core.logging import get_logger
+from additory.common.validation import validate_dataframe
+from additory.common.result import wrap_analysis
+
+from . import quality
+from . import cardinality
+from . import distributions
+from . import correlations
+from . import types
+from . import features
+from . import patterns
+from . import outliers
+from . import duplicates
+from . import timeseries
+from . import imputation
+from . import presets
+
+
+logger = get_logger()
+
+
+def analyze(
+    df,
+    preset: Optional[str] = None,
+    quality_analysis: bool = False,
+    cardinality_analysis: bool = False,
+    distributions_analysis: bool = False,
+    correlations_analysis: bool = False,
+    features_analysis: bool = False,
+    types_analysis: bool = False,
+    patterns_analysis: bool = False,
+    outliers_analysis: bool = False,
+    duplicates_analysis: bool = False,
+    timeseries_analysis: bool = False,
+    imputation_analysis: bool = False,
+    date_column: Optional[str] = None
+):
+    """
+    Analyze DataFrame with multiple analysis types.
+    
+    Args:
+        df: Input DataFrame
+        preset: Preset name ('quick', 'full')
+        quality_analysis: Run quality analysis
+        cardinality_analysis: Run cardinality analysis
+        distributions_analysis: Run distribution analysis
+        correlations_analysis: Run correlation analysis
+        features_analysis: Run feature analysis
+        types_analysis: Run type analysis
+        patterns_analysis: Run pattern analysis
+        outliers_analysis: Run outlier analysis
+        duplicates_analysis: Run duplicate analysis
+        timeseries_analysis: Run timeseries analysis
+        imputation_analysis: Run imputation analysis
+        date_column: Column for timeseries analysis
+        
+    Returns:
+        AnalysisResult with analysis results
+    """
+    start_time = time.time()
+    
+    # Validate and convert
+    validate_dataframe(df, 'df')
+    backend = detect_backend(df)
+    polars_df = to_polars(df)
+    
+    # Determine analyses to run
+    if preset:
+        analyses_to_run = presets.get_preset_analyses(preset)
+    else:
+        analyses_to_run = {
+            'quality': quality_analysis,
+            'cardinality': cardinality_analysis,
+            'distributions': distributions_analysis,
+            'correlations': correlations_analysis,
+            'features': features_analysis,
+            'types': types_analysis,
+            'patterns': patterns_analysis,
+            'outliers': outliers_analysis,
+            'duplicates': duplicates_analysis,
+            'timeseries': timeseries_analysis,
+            'imputation': imputation_analysis
+        }
+    
+    # Run analyses
+    results = {}
+    
+    if analyses_to_run.get('quality'):
+        results['quality'] = quality.analyze_quality(polars_df)
+    
+    if analyses_to_run.get('cardinality'):
+        results['cardinality'] = cardinality.analyze_cardinality(polars_df)
+    
+    if analyses_to_run.get('distributions'):
+        results['distributions'] = distributions.analyze_distributions(polars_df)
+    
+    if analyses_to_run.get('correlations'):
+        results['correlations'] = correlations.analyze_correlations(polars_df)
+    
+    if analyses_to_run.get('features'):
+        results['features'] = features.analyze_features(polars_df)
+    
+    if analyses_to_run.get('types'):
+        results['types'] = types.analyze_types(polars_df)
+    
+    if analyses_to_run.get('patterns'):
+        results['patterns'] = patterns.analyze_patterns(polars_df)
+    
+    if analyses_to_run.get('outliers'):
+        results['outliers'] = outliers.analyze_outliers(polars_df)
+    
+    if analyses_to_run.get('duplicates'):
+        results['duplicates'] = duplicates.analyze_duplicates(polars_df)
+    
+    if analyses_to_run.get('timeseries') and date_column:
+        results['timeseries'] = timeseries.analyze_timeseries(polars_df, date_column)
+    
+    if analyses_to_run.get('imputation'):
+        results['imputation'] = imputation.analyze_imputation(polars_df)
+    
+    # Calculate execution time
+    execution_time = time.time() - start_time
+    
+    # Wrap and return
+    metadata = {
+        'preset': preset,
+        'analyses_run': list(results.keys()),
+        'execution_time': execution_time,
+        'input_shape': (polars_df.height, polars_df.width)
+    }
+    
+    return wrap_analysis(results, metadata)

additory/functions/analyze/cardinality.py

@@ -0,0 +1,58 @@
+"""
+Cardinality analysis module.
+
+Analyzes unique value counts and cardinality ratios to identify
+potential keys and categorical columns.
+"""
+
+import polars as pl
+from typing import Dict, Any, List
+
+
+def analyze_cardinality(df: pl.DataFrame) -> Dict[str, Any]:
+    """
+    Analyze cardinality metrics.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        
+    Returns:
+        Dictionary with cardinality metrics:
+        - unique_counts: Dict of column -> unique count
+        - cardinality_ratios: Dict of column -> ratio (unique/total)
+        - high_cardinality: List of high cardinality columns (ratio > 0.95)
+        - low_cardinality: List of low cardinality columns (ratio < 0.05)
+        - potential_keys: List of columns that could be keys (ratio == 1.0)
+        - potential_categories: List of columns that could be categories (ratio < 0.05)
+        
+    Example:
+        >>> df = pl.DataFrame({'id': [1, 2, 3], 'category': ['A', 'A', 'B']})
+        >>> result = analyze_cardinality(df)
+        >>> result['potential_keys']
+        ['id']
+    """
+    total_rows = df.height
+    
+    # Calculate unique counts and ratios
+    unique_counts = {}
+    cardinality_ratios = {}
+    
+    for col in df.columns:
+        unique_count = df[col].n_unique()
+        unique_counts[col] = unique_count
+        cardinality_ratios[col] = (unique_count / total_rows) if total_rows > 0 else 0.0
+    
+    # Classify columns by cardinality
+    high_cardinality = [col for col, ratio in cardinality_ratios.items() if ratio > 0.95]
+    low_cardinality = [col for col, ratio in cardinality_ratios.items() if ratio < 0.05]
+    potential_keys = [col for col, ratio in cardinality_ratios.items() if ratio == 1.0]
+    potential_categories = [col for col, ratio in cardinality_ratios.items() if ratio < 0.05 and ratio > 0]
+    
+    return {
+        'unique_counts': unique_counts,
+        'cardinality_ratios': {k: round(v, 4) for k, v in cardinality_ratios.items()},
+        'high_cardinality': high_cardinality,
+        'low_cardinality': low_cardinality,
+        'potential_keys': potential_keys,
+        'potential_categories': potential_categories
+    }

additory/functions/analyze/correlations.py

@@ -0,0 +1,66 @@
+"""
+Correlation analysis module.
+
+Analyzes correlations between numeric columns using Pearson correlation.
+"""
+
+import polars as pl
+from typing import Dict, Any, List, Tuple
+
+
+def analyze_correlations(df: pl.DataFrame) -> Dict[str, Any]:
+    """
+    Analyze correlations between numeric columns.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        
+    Returns:
+        Dictionary with correlation metrics:
+        - correlation_matrix: Dict of column pairs -> correlation value
+        - highly_correlated: List of highly correlated pairs (|r| > 0.7)
+        - numeric_columns: List of columns analyzed
+        
+    Example:
+        >>> df = pl.DataFrame({'a': [1, 2, 3], 'b': [2, 4, 6], 'c': [1, 1, 1]})
+        >>> result = analyze_correlations(df)
+        >>> len(result['highly_correlated']) > 0
+        True
+    """
+    # Get numeric columns
+    numeric_cols = [col for col in df.columns if df[col].dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, pl.Float32, pl.Float64]]
+    
+    if len(numeric_cols) < 2:
+        return {
+            'correlation_matrix': {},
+            'highly_correlated': [],
+            'numeric_columns': numeric_cols
+        }
+    
+    # Calculate correlation matrix
+    correlation_matrix = {}
+    highly_correlated = []
+    
+    for i, col1 in enumerate(numeric_cols):
+        for col2 in numeric_cols[i+1:]:
+            # Calculate Pearson correlation
+            corr = df.select([
+                pl.corr(col1, col2).alias('correlation')
+            ])['correlation'][0]
+            
+            if corr is not None:
+                correlation_matrix[f"{col1}_{col2}"] = round(float(corr), 4)
+                
+                # Check if highly correlated
+                if abs(corr) > 0.7:
+                    highly_correlated.append({
+                        'column1': col1,
+                        'column2': col2,
+                        'correlation': round(float(corr), 4)
+                    })
+    
+    return {
+        'correlation_matrix': correlation_matrix,
+        'highly_correlated': highly_correlated,
+        'numeric_columns': numeric_cols
+    }

additory/functions/analyze/distributions.py

@@ -0,0 +1,53 @@
+"""
+Distribution analysis module.
+
+Analyzes statistical distributions of numeric columns including
+mean, median, std, skewness, and kurtosis.
+"""
+
+import polars as pl
+from typing import Dict, Any
+
+
+def analyze_distributions(df: pl.DataFrame) -> Dict[str, Any]:
+    """
+    Analyze distribution metrics for numeric columns.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        
+    Returns:
+        Dictionary with distribution metrics per numeric column:
+        - mean: Mean value
+        - median: Median value
+        - std: Standard deviation
+        - min: Minimum value
+        - max: Maximum value
+        - q25: 25th percentile
+        - q75: 75th percentile
+        
+    Example:
+        >>> df = pl.DataFrame({'a': [1, 2, 3, 4, 5], 'b': ['x', 'y', 'z', 'x', 'y']})
+        >>> result = analyze_distributions(df)
+        >>> 'a' in result
+        True
+    """
+    numeric_cols = [col for col in df.columns if df[col].dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, pl.Float32, pl.Float64]]
+    
+    distributions = {}
+    
+    for col in numeric_cols:
+        col_data = df[col]
+        
+        # Calculate basic statistics
+        distributions[col] = {
+            'mean': float(col_data.mean()) if col_data.mean() is not None else None,
+            'median': float(col_data.median()) if col_data.median() is not None else None,
+            'std': float(col_data.std()) if col_data.std() is not None else None,
+            'min': float(col_data.min()) if col_data.min() is not None else None,
+            'max': float(col_data.max()) if col_data.max() is not None else None,
+            'q25': float(col_data.quantile(0.25)) if col_data.quantile(0.25) is not None else None,
+            'q75': float(col_data.quantile(0.75)) if col_data.quantile(0.75) is not None else None
+        }
+    
+    return distributions

additory/functions/analyze/duplicates.py

@@ -0,0 +1,49 @@
+"""
+Duplicate detection module.
+
+Detects duplicate rows and values in the DataFrame.
+"""
+
+import polars as pl
+from typing import Dict, Any
+
+
+def analyze_duplicates(df: pl.DataFrame) -> Dict[str, Any]:
+    """
+    Detect duplicate rows and values.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        
+    Returns:
+        Dictionary with duplicate detection results:
+        - duplicate_rows: Number of duplicate rows
+        - duplicate_percentage: Percentage of duplicate rows
+        - columns_with_duplicates: List of columns with duplicate values
+        - unique_rows: Number of unique rows
+        
+    Example:
+        >>> df = pl.DataFrame({'a': [1, 1, 2], 'b': [1, 1, 2]})
+        >>> result = analyze_duplicates(df)
+        >>> result['duplicate_rows']
+        1
+    """
+    total_rows = df.height
+    unique_rows = df.unique().height
+    duplicate_rows = total_rows - unique_rows
+    duplicate_percentage = (duplicate_rows / total_rows * 100) if total_rows > 0 else 0.0
+    
+    # Check for duplicates in each column
+    columns_with_duplicates = []
+    for col in df.columns:
+        unique_count = df[col].n_unique()
+        if unique_count < total_rows:
+            columns_with_duplicates.append(col)
+    
+    return {
+        'duplicate_rows': duplicate_rows,
+        'duplicate_percentage': round(duplicate_percentage, 2),
+        'unique_rows': unique_rows,
+        'total_rows': total_rows,
+        'columns_with_duplicates': columns_with_duplicates
+    }