additory 0.1.0a4__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/functions/analyze/features.py

@@ -0,0 +1,61 @@
+"""
+Feature analysis module.
+
+Analyzes feature types and provides transformation recommendations.
+"""
+
+import polars as pl
+from typing import Dict, Any
+
+
+def analyze_features(df: pl.DataFrame) -> Dict[str, Any]:
+    """
+    Analyze features and provide recommendations.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        
+    Returns:
+        Dictionary with feature analysis:
+        - feature_types: Dict of column -> feature type
+        - numeric_features: List of numeric features
+        - categorical_features: List of categorical features
+        - datetime_features: List of datetime features
+        
+    Example:
+        >>> df = pl.DataFrame({'a': [1, 2, 3], 'b': ['x', 'y', 'z']})
+        >>> result = analyze_features(df)
+        >>> result['numeric_features']
+        ['a']
+    """
+    feature_types = {}
+    numeric_features = []
+    categorical_features = []
+    datetime_features = []
+    
+    for col in df.columns:
+        dtype = df[col].dtype
+        
+        if dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, pl.Float32, pl.Float64]:
+            feature_types[col] = 'numeric'
+            numeric_features.append(col)
+        elif dtype == pl.Utf8:
+            # Check if it's categorical (low cardinality)
+            unique_ratio = df[col].n_unique() / df.height if df.height > 0 else 0
+            if unique_ratio < 0.05:
+                feature_types[col] = 'categorical'
+                categorical_features.append(col)
+            else:
+                feature_types[col] = 'text'
+        elif dtype in [pl.Date, pl.Datetime]:
+            feature_types[col] = 'datetime'
+            datetime_features.append(col)
+        else:
+            feature_types[col] = str(dtype)
+    
+    return {
+        'feature_types': feature_types,
+        'numeric_features': numeric_features,
+        'categorical_features': categorical_features,
+        'datetime_features': datetime_features
+    }

additory/functions/analyze/imputation.py

@@ -0,0 +1,66 @@
+"""
+Imputation recommendation module.
+
+Recommends imputation strategies for missing values.
+"""
+
+import polars as pl
+from typing import Dict, Any
+
+
+def analyze_imputation(df: pl.DataFrame) -> Dict[str, Any]:
+    """
+    Recommend imputation strategies for missing values.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        
+    Returns:
+        Dictionary with imputation recommendations:
+        - recommendations: Dict of column -> recommended strategy
+        - columns_needing_imputation: List of columns with missing values
+        - imputation_complexity: Overall complexity (low, medium, high)
+        
+    Example:
+        >>> df = pl.DataFrame({'a': [1, None, 3], 'b': ['x', 'y', None]})
+        >>> result = analyze_imputation(df)
+        >>> 'a' in result['columns_needing_imputation']
+        True
+    """
+    recommendations = {}
+    columns_needing_imputation = []
+    
+    for col in df.columns:
+        null_count = df[col].null_count()
+        
+        if null_count == 0:
+            continue
+        
+        columns_needing_imputation.append(col)
+        dtype = df[col].dtype
+        
+        # Recommend strategy based on type
+        if dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, pl.Float32, pl.Float64]:
+            recommendations[col] = 'mean or median'
+        elif dtype == pl.Utf8:
+            recommendations[col] = 'mode or constant'
+        elif dtype == pl.Boolean:
+            recommendations[col] = 'mode'
+        else:
+            recommendations[col] = 'forward fill or constant'
+    
+    # Determine complexity
+    if len(columns_needing_imputation) == 0:
+        complexity = 'none'
+    elif len(columns_needing_imputation) <= 2:
+        complexity = 'low'
+    elif len(columns_needing_imputation) <= 5:
+        complexity = 'medium'
+    else:
+        complexity = 'high'
+    
+    return {
+        'recommendations': recommendations,
+        'columns_needing_imputation': columns_needing_imputation,
+        'imputation_complexity': complexity
+    }

additory/functions/analyze/outliers.py

@@ -0,0 +1,65 @@
+"""
+Outlier detection module.
+
+Detects outliers in numeric columns using IQR method.
+"""
+
+import polars as pl
+from typing import Dict, Any
+
+
+def analyze_outliers(df: pl.DataFrame) -> Dict[str, Any]:
+    """
+    Detect outliers in numeric columns using IQR method.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        
+    Returns:
+        Dictionary with outlier detection results:
+        - outlier_counts: Dict of column -> outlier count
+        - outlier_percentages: Dict of column -> outlier percentage
+        - columns_with_outliers: List of columns with outliers
+        
+    Example:
+        >>> df = pl.DataFrame({'a': [1, 2, 3, 4, 100]})
+        >>> result = analyze_outliers(df)
+        >>> result['outlier_counts']['a'] > 0
+        True
+    """
+    numeric_cols = [col for col in df.columns if df[col].dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, pl.Float32, pl.Float64]]
+    
+    outlier_counts = {}
+    outlier_percentages = {}
+    
+    for col in numeric_cols:
+        col_data = df[col].drop_nulls()
+        
+        if len(col_data) == 0:
+            outlier_counts[col] = 0
+            outlier_percentages[col] = 0.0
+            continue
+        
+        # Calculate IQR
+        q25 = col_data.quantile(0.25)
+        q75 = col_data.quantile(0.75)
+        iqr = q75 - q25
+        
+        # Calculate bounds
+        lower_bound = q25 - 1.5 * iqr
+        upper_bound = q75 + 1.5 * iqr
+        
+        # Count outliers
+        outliers = col_data.filter((col_data < lower_bound) | (col_data > upper_bound))
+        outlier_count = len(outliers)
+        
+        outlier_counts[col] = outlier_count
+        outlier_percentages[col] = (outlier_count / len(col_data) * 100) if len(col_data) > 0 else 0.0
+    
+    columns_with_outliers = [col for col, count in outlier_counts.items() if count > 0]
+    
+    return {
+        'outlier_counts': outlier_counts,
+        'outlier_percentages': {k: round(v, 2) for k, v in outlier_percentages.items()},
+        'columns_with_outliers': columns_with_outliers
+    }

additory/functions/analyze/patterns.py

@@ -0,0 +1,65 @@
+"""
+Pattern detection module.
+
+Detects common patterns in string columns like emails, phone numbers, etc.
+"""
+
+import polars as pl
+import re
+from typing import Dict, Any
+
+
+def analyze_patterns(df: pl.DataFrame) -> Dict[str, Any]:
+    """
+    Detect patterns in string columns.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        
+    Returns:
+        Dictionary with pattern detection results:
+        - email_columns: List of columns containing emails
+        - phone_columns: List of columns containing phone numbers
+        - date_string_columns: List of columns containing date strings
+        - id_columns: List of columns containing IDs
+        
+    Example:
+        >>> df = pl.DataFrame({'email': ['a@b.com', 'c@d.com']})
+        >>> result = analyze_patterns(df)
+        >>> 'email' in result['email_columns']
+        True
+    """
+    email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+    phone_pattern = r'^\+?[\d\s\-\(\)]{10,}$'
+    
+    email_columns = []
+    phone_columns = []
+    date_string_columns = []
+    id_columns = []
+    
+    for col in df.columns:
+        if df[col].dtype != pl.Utf8:
+            continue
+        
+        # Sample first 100 non-null values
+        sample = df[col].drop_nulls().head(100).to_list()
+        
+        if not sample:
+            continue
+        
+        # Check for email pattern
+        email_matches = sum(1 for val in sample if re.match(email_pattern, str(val)))
+        if email_matches / len(sample) > 0.8:
+            email_columns.append(col)
+        
+        # Check for phone pattern
+        phone_matches = sum(1 for val in sample if re.match(phone_pattern, str(val)))
+        if phone_matches / len(sample) > 0.8:
+            phone_columns.append(col)
+    
+    return {
+        'email_columns': email_columns,
+        'phone_columns': phone_columns,
+        'date_string_columns': date_string_columns,
+        'id_columns': id_columns
+    }

additory/functions/analyze/presets.py

@@ -0,0 +1,72 @@
+"""
+Preset analysis configurations.
+
+Provides preset analysis configurations for quick and full analysis.
+"""
+
+from typing import Dict, List
+
+
+def get_preset_analyses(preset_name: str) -> Dict[str, bool]:
+    """
+    Get analyses for a preset.
+    
+    Args:
+        preset_name: Name of preset ('quick' or 'full')
+        
+    Returns:
+        Dictionary of analysis_name -> enabled
+        
+    Example:
+        >>> analyses = get_preset_analyses('quick')
+        >>> analyses['quality']
+        True
+    """
+    presets = {
+        'quick': {
+            'quality': True,
+            'cardinality': True,
+            'types': True,
+            'distributions': False,
+            'correlations': False,
+            'features': False,
+            'patterns': False,
+            'outliers': False,
+            'duplicates': False,
+            'timeseries': False,
+            'imputation': False
+        },
+        'full': {
+            'quality': True,
+            'cardinality': True,
+            'types': True,
+            'distributions': True,
+            'correlations': True,
+            'features': True,
+            'patterns': True,
+            'outliers': True,
+            'duplicates': True,
+            'timeseries': False,  # Requires date_column
+            'imputation': True
+        }
+    }
+    
+    if preset_name not in presets:
+        raise ValueError(f"Unknown preset '{preset_name}'. Available: {list(presets.keys())}")
+    
+    return presets[preset_name]
+
+
+def list_presets() -> List[str]:
+    """
+    List available presets.
+    
+    Returns:
+        List of preset names
+        
+    Example:
+        >>> presets = list_presets()
+        >>> 'quick' in presets
+        True
+    """
+    return ['quick', 'full']

additory/functions/analyze/quality.py

@@ -0,0 +1,59 @@
+"""
+Data quality analysis module.
+
+Analyzes data quality metrics including missing values, type consistency,
+and format violations.
+"""
+
+import polars as pl
+from typing import Dict, Any
+
+
+def analyze_quality(df: pl.DataFrame) -> Dict[str, Any]:
+    """
+    Analyze data quality metrics.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        
+    Returns:
+        Dictionary with quality metrics:
+        - missing_values: Dict of column -> count
+        - missing_percentages: Dict of column -> percentage
+        - total_rows: Total number of rows
+        - columns_with_nulls: List of columns with missing values
+        - quality_score: Overall quality score (0-100)
+        
+    Example:
+        >>> df = pl.DataFrame({'a': [1, 2, None], 'b': [1, 2, 3]})
+        >>> result = analyze_quality(df)
+        >>> result['missing_values']
+        {'a': 1, 'b': 0}
+    """
+    total_rows = df.height
+    total_cells = total_rows * df.width
+    
+    # Calculate missing values per column
+    missing_values = {}
+    missing_percentages = {}
+    
+    for col in df.columns:
+        null_count = df[col].null_count()
+        missing_values[col] = null_count
+        missing_percentages[col] = (null_count / total_rows * 100) if total_rows > 0 else 0.0
+    
+    # Identify columns with nulls
+    columns_with_nulls = [col for col, count in missing_values.items() if count > 0]
+    
+    # Calculate overall quality score (percentage of non-null cells)
+    total_nulls = sum(missing_values.values())
+    quality_score = ((total_cells - total_nulls) / total_cells * 100) if total_cells > 0 else 100.0
+    
+    return {
+        'missing_values': missing_values,
+        'missing_percentages': missing_percentages,
+        'total_rows': total_rows,
+        'total_columns': df.width,
+        'columns_with_nulls': columns_with_nulls,
+        'quality_score': round(quality_score, 2)
+    }

additory/functions/analyze/timeseries.py

@@ -0,0 +1,53 @@
+"""
+Time series analysis module.
+
+Analyzes time series data for trends and patterns.
+"""
+
+import polars as pl
+from typing import Dict, Any, Optional
+
+
+def analyze_timeseries(df: pl.DataFrame, date_column: str) -> Dict[str, Any]:
+    """
+    Analyze time series data.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        date_column: Column containing dates
+        
+    Returns:
+        Dictionary with timeseries analysis:
+        - date_column: Name of date column
+        - date_range: Tuple of (min_date, max_date)
+        - total_days: Number of days in range
+        - row_count: Number of rows
+        - frequency: Estimated frequency (daily, weekly, etc.)
+        
+    Example:
+        >>> df = pl.DataFrame({'date': ['2024-01-01', '2024-01-02'], 'value': [1, 2]})
+        >>> df = df.with_columns(pl.col('date').str.strptime(pl.Date, '%Y-%m-%d'))
+        >>> result = analyze_timeseries(df, 'date')
+        >>> result['row_count']
+        2
+    """
+    if date_column not in df.columns:
+        raise ValueError(f"Column '{date_column}' not found in DataFrame")
+    
+    # Get date range
+    min_date = df[date_column].min()
+    max_date = df[date_column].max()
+    
+    # Calculate total days
+    if min_date is not None and max_date is not None:
+        total_days = (max_date - min_date).days if hasattr(max_date - min_date, 'days') else 0
+    else:
+        total_days = 0
+    
+    return {
+        'date_column': date_column,
+        'date_range': (str(min_date), str(max_date)),
+        'total_days': total_days,
+        'row_count': df.height,
+        'frequency': 'unknown'
+    }

additory/functions/analyze/types.py

@@ -0,0 +1,45 @@
+"""
+Data type analysis module.
+
+Analyzes data types and provides type recommendations.
+"""
+
+import polars as pl
+from typing import Dict, Any
+
+
+def analyze_types(df: pl.DataFrame) -> Dict[str, Any]:
+    """
+    Analyze data types of columns.
+    
+    Args:
+        df: Polars DataFrame to analyze
+        
+    Returns:
+        Dictionary with type information:
+        - column_types: Dict of column -> type name
+        - numeric_columns: List of numeric columns
+        - string_columns: List of string columns
+        - boolean_columns: List of boolean columns
+        - date_columns: List of date/datetime columns
+        
+    Example:
+        >>> df = pl.DataFrame({'a': [1, 2, 3], 'b': ['x', 'y', 'z']})
+        >>> result = analyze_types(df)
+        >>> result['numeric_columns']
+        ['a']
+    """
+    column_types = {col: str(df[col].dtype) for col in df.columns}
+    
+    numeric_columns = [col for col in df.columns if df[col].dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, pl.Float32, pl.Float64]]
+    string_columns = [col for col in df.columns if df[col].dtype == pl.Utf8]
+    boolean_columns = [col for col in df.columns if df[col].dtype == pl.Boolean]
+    date_columns = [col for col in df.columns if df[col].dtype in [pl.Date, pl.Datetime]]
+    
+    return {
+        'column_types': column_types,
+        'numeric_columns': numeric_columns,
+        'string_columns': string_columns,
+        'boolean_columns': boolean_columns,
+        'date_columns': date_columns
+    }

additory/functions/expressions/__init__.py

@@ -0,0 +1,161 @@
+"""
+Expressions function - Evaluate expressions and add result columns.
+
+Supports both inline expressions and references to named expressions.
+"""
+
+import polars as pl
+import time
+import re
+from typing import List
+
+from additory.core.backend import detect_backend, to_polars, from_polars
+from additory.core.logging import get_logger
+from additory.common.validation import validate_dataframe, validate_not_empty
+from additory.common.result import wrap_result
+from additory.expressions.engine import get_engine
+
+
+logger = get_logger()
+
+
+def expressions(df, *expressions_list):
+    """
+    Evaluate one or more expressions and add result columns.
+    
+    Args:
+        df: Input DataFrame
+        *expressions_list: Variable number of expression strings
+        
+    Returns:
+        DataFrameResult with computed columns
+        
+    Examples:
+        >>> # Single expression
+        >>> result = expressions(df, 'inbuilt:bmi')
+        
+        >>> # Multiple expressions
+        >>> result = expressions(df, 'inbuilt:bmi', 'inbuilt:bsa')
+        
+        >>> # Inline expression
+        >>> result = expressions(df, 'weight / (height ** 2)')
+        
+        >>> # Mix of inline and references
+        >>> result = expressions(df, 'inbuilt:bmi', 'age * 12')
+    """
+    start_time = time.time()
+    
+    # Validate inputs
+    validate_dataframe(df, 'df')
+    validate_not_empty(df, 'df')
+    
+    if not expressions_list:
+        raise ValueError("At least one expression required")
+    
+    # Detect backend and convert to Polars
+    backend = detect_backend(df)
+    polars_df = to_polars(df)
+    
+    # Get expression engine
+    engine = get_engine()
+    
+    # Track columns added
+    columns_added = []
+    
+    # Evaluate each expression
+    result_df = polars_df
+    for expr_str in expressions_list:
+        # Evaluate expression
+        result_series = engine.evaluate(result_df, expr_str)
+        
+        # Determine column name
+        col_name = determine_column_name(expr_str)
+        
+        # Add to DataFrame
+        result_df = result_df.with_columns(result_series.alias(col_name))
+        columns_added.append(col_name)
+        
+        logger.info(f"Added column '{col_name}' from expression '{expr_str}'")
+    
+    # Convert back to original backend
+    result_df = from_polars(result_df, backend)
+    
+    # Calculate execution time
+    execution_time = time.time() - start_time
+    
+    # Wrap result
+    metadata = {
+        'expressions_evaluated': len(expressions_list),
+        'columns_added': columns_added,
+        'input_shape': (polars_df.height, polars_df.width),
+        'execution_time': execution_time
+    }
+    
+    return wrap_result(result_df, 'expressions', metadata)
+
+
+def determine_column_name(expression: str) -> str:
+    """
+    Determine column name for expression result.
+    
+    Args:
+        expression: Expression string
+        
+    Returns:
+        Column name
+        
+    Examples:
+        >>> determine_column_name('inbuilt:bmi')
+        'bmi'
+        >>> determine_column_name('weight / height')
+        'weight_div_height'
+    """
+    # If reference (e.g., 'inbuilt:bmi'), use expression name
+    if ':' in expression:
+        parts = expression.split(':', 1)
+        if len(parts) == 2 and parts[0] in ['inbuilt', 'user', 'company']:
+            return parts[1]
+    
+    # If inline, infer from expression
+    return infer_column_name(expression)
+
+
+def infer_column_name(expression: str) -> str:
+    """
+    Infer column name from inline expression.
+    
+    Args:
+        expression: Inline expression string
+        
+    Returns:
+        Inferred column name
+        
+    Examples:
+        >>> infer_column_name('weight / height')
+        'weight_div_height'
+        >>> infer_column_name('sqrt(age)')
+        'sqrt_age'
+        >>> infer_column_name('age * 12')
+        'age_mul_12'
+    """
+    # Replace operators with words
+    name = expression
+    name = name.replace(' / ', '_div_')
+    name = name.replace(' * ', '_mul_')
+    name = name.replace(' + ', '_add_')
+    name = name.replace(' - ', '_sub_')
+    name = name.replace('**', '_pow_')
+    
+    # Remove parentheses and spaces
+    name = name.replace('(', '_')
+    name = name.replace(')', '')
+    name = name.replace(' ', '')
+    
+    # Remove trailing underscores
+    name = name.strip('_')
+    
+    # If name is too long or complex, use generic name
+    if len(name) > 50 or not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name):
+        return 'expr_result'
+    
+    return name