additory 0.1.0a3__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/functions/to/summarize.py

@@ -0,0 +1,170 @@
+"""
+Group and aggregate data.
+
+This module provides summarization functionality for the to function.
+"""
+
+import polars as pl
+from typing import Union, List, Dict
+
+from additory.common.validation import validate_dataframe, validate_not_empty
+from additory.core.logging import Logger
+
+
+# Mapping of aggregation names to Polars methods
+AGGREGATION_MAP = {
+    'sum': 'sum',
+    'mean': 'mean',
+    'median': 'median',
+    'min': 'min',
+    'max': 'max',
+    'count': 'count',
+    'count_unique': 'n_unique',
+    'std': 'std',
+    'var': 'var',
+    'first': 'first',
+    'last': 'last',
+}
+
+
+def perform_summarize(
+    df: pl.DataFrame,
+    group_by: Union[str, List[str]],
+    aggregations: Dict[str, str],
+    **kwargs
+) -> pl.DataFrame:
+    """
+    Group and aggregate DataFrame.
+    
+    Args:
+        df: Input DataFrame
+        group_by: Column(s) to group by
+        aggregations: Dictionary mapping column to aggregation function
+        **kwargs: Additional parameters (reserved for future use)
+        
+    Returns:
+        Summarized DataFrame
+        
+    Example:
+        >>> result = perform_summarize(
+        ...     df,
+        ...     group_by='category',
+        ...     aggregations={'sales': 'sum', 'orders': 'count', 'price': 'mean'}
+        ... )
+        
+    Supported Aggregations:
+        - 'sum', 'mean', 'median', 'min', 'max'
+        - 'count', 'count_unique', 'std', 'var'
+        - 'first', 'last'
+    """
+    logger = Logger()
+    
+    # Validate parameters
+    validate_summarize_parameters(df, group_by, aggregations)
+    
+    # Normalize group_by to list
+    group_by_list = [group_by] if isinstance(group_by, str) else group_by
+    
+    # Log operation
+    logger.info(f"Summarizing by {group_by_list}, {len(aggregations)} aggregations")
+    
+    # Parse aggregation specifications
+    agg_exprs = parse_aggregation_spec(aggregations)
+    
+    # Perform groupby and aggregation
+    result = df.group_by(group_by_list).agg(agg_exprs)
+    
+    logger.info(f"Summarize complete: {len(result)} groups")
+    
+    return result
+
+
+def validate_summarize_parameters(
+    df: pl.DataFrame,
+    group_by: Union[str, List[str]],
+    aggregations: Dict[str, str]
+) -> None:
+    """
+    Validate summarize parameters.
+    
+    Args:
+        df: Input DataFrame
+        group_by: Column(s) to group by
+        aggregations: Dictionary mapping column to aggregation function
+        
+    Raises:
+        ValueError: If validation fails
+    """
+    # Validate DataFrame
+    validate_dataframe(df)
+    validate_not_empty(df)
+    
+    # Normalize group_by to list
+    group_by_list = [group_by] if isinstance(group_by, str) else group_by
+    
+    # Check that group_by columns exist
+    missing_cols = [col for col in group_by_list if col not in df.columns]
+    if missing_cols:
+        raise ValueError(
+            f"Group by columns not found in DataFrame: {missing_cols}. "
+            f"Available columns: {df.columns}"
+        )
+    
+    # Check that aggregations is not empty
+    if not aggregations:
+        raise ValueError("Aggregations dictionary cannot be empty")
+    
+    # Check that aggregation columns exist
+    missing_agg_cols = [col for col in aggregations.keys() if col not in df.columns]
+    if missing_agg_cols:
+        raise ValueError(
+            f"Aggregation columns not found in DataFrame: {missing_agg_cols}. "
+            f"Available columns: {df.columns}"
+        )
+    
+    # Check that aggregation functions are valid
+    invalid_funcs = [
+        func for func in aggregations.values() 
+        if func not in AGGREGATION_MAP
+    ]
+    if invalid_funcs:
+        raise ValueError(
+            f"Invalid aggregation functions: {invalid_funcs}. "
+            f"Valid functions: {list(AGGREGATION_MAP.keys())}"
+        )
+
+
+def parse_aggregation_spec(aggregations: Dict[str, str]) -> List[pl.Expr]:
+    """
+    Parse aggregation specifications into Polars expressions.
+    
+    Args:
+        aggregations: Dictionary mapping column to aggregation function
+        
+    Returns:
+        List of Polars expressions
+        
+    Example:
+        >>> # Input: {'sales': 'sum', 'orders': 'count'}
+        >>> # Output: [pl.col('sales').sum(), pl.col('orders').count()]
+    """
+    exprs = []
+    
+    for col, func in aggregations.items():
+        # Get the Polars method name
+        polars_method = AGGREGATION_MAP[func]
+        
+        # Create expression
+        if polars_method == 'n_unique':
+            # Special case for count_unique
+            expr = pl.col(col).n_unique()
+        elif polars_method == 'count':
+            # For count, we count non-null values
+            expr = pl.col(col).count()
+        else:
+            # Standard aggregation
+            expr = getattr(pl.col(col), polars_method)()
+        
+        exprs.append(expr)
+    
+    return exprs

additory/functions/transform/__init__.py

@@ -0,0 +1,140 @@
+"""
+Main transform function - transform DataFrame structure.
+
+This module provides the main user-facing transform() function with 6 modes:
+- transpose: Transpose DataFrame (rows ↔ columns)
+- onehotencoding: One-hot encode categorical columns
+- extract: Extract features from columns
+- datetime: Normalize datetime columns
+- harmonize: Harmonize units across columns
+- knn: Impute missing values using KNN
+"""
+
+import polars as pl
+from typing import Any, Optional, Dict, List, Union
+
+from additory.core.backend import detect_backend, to_polars, from_polars
+from additory.core.logging import Logger
+from additory.core.memory_manager import MemoryManager
+from additory.common.validation import validate_dataframe, validate_not_empty
+from additory.common.result import wrap_result
+
+from additory.functions.transform.transpose import perform_transpose
+from additory.functions.transform.onehotencoding import perform_onehotencoding
+from additory.functions.transform.extract import perform_extract
+from additory.functions.transform.datetime import perform_datetime_normalization
+from additory.functions.transform.harmonize import perform_harmonize
+from additory.functions.transform.knn import perform_knn_imputation
+
+
+def transform(
+    df: Any,
+    mode: str,
+    columns: Optional[Union[str, List[str], Dict]] = None,
+    strategy: Optional[Dict] = None,
+    **kwargs
+) -> Any:
+    """
+    Transform DataFrame structure using various modes.
+    
+    Args:
+        df: Input DataFrame
+        mode: Transformation mode
+        columns: Columns to transform (mode-specific)
+        strategy: Strategy dictionary for advanced control
+        **kwargs: Mode-specific parameters
+        
+    Returns:
+        Transformed DataFrame (wrapped in Result)
+        
+    Modes:
+        1. transpose: Transpose DataFrame (rows ↔ columns)
+           - Example: transform(df, mode='transpose')
+           
+        2. onehotencoding: One-hot encode categorical columns
+           - Example: transform(df, mode='onehotencoding', columns=['category'])
+           
+        3. extract: Extract features from columns
+           - Example: transform(df, mode='extract', columns={'date': ['hour', 'day']})
+           
+        4. datetime: Normalize datetime columns
+           - Example: transform(df, mode='datetime', columns=['birth_date'], strategy={...})
+           
+        5. harmonize: Harmonize units across columns
+           - Example: transform(df, mode='harmonize', columns=['height'], strategy={...})
+           
+        6. knn: Impute missing values using KNN
+           - Example: transform(df, mode='knn', columns=['age'], strategy={'k': 5})
+    """
+    logger = Logger()
+    memory_manager = MemoryManager()
+    
+    try:
+        # Validate input
+        validate_dataframe(df)
+        validate_not_empty(df)
+        
+        # Detect backend and convert to Polars
+        backend = detect_backend(df)
+        polars_df = to_polars(df)
+        
+        # Set logging context
+        logger.set_context('transform', {'mode': mode})
+        logger.info(f"Starting transform() function in '{mode}' mode")
+        
+        # Dispatch to appropriate mode
+        if mode == 'transpose':
+            result = perform_transpose(polars_df)
+        
+        elif mode == 'onehotencoding':
+            if columns is None:
+                raise ValueError("columns parameter required for onehotencoding mode")
+            result = perform_onehotencoding(polars_df, columns)
+        
+        elif mode == 'extract':
+            if columns is None:
+                raise ValueError("columns parameter required for extract mode")
+            if not isinstance(columns, dict):
+                raise TypeError("columns must be a dictionary for extract mode")
+            result = perform_extract(polars_df, columns, strategy)
+        
+        elif mode == 'datetime':
+            if columns is None:
+                raise ValueError("columns parameter required for datetime mode")
+            columns_list = [columns] if isinstance(columns, str) else columns
+            result = perform_datetime_normalization(polars_df, columns_list, strategy)
+        
+        elif mode == 'harmonize':
+            if columns is None:
+                raise ValueError("columns parameter required for harmonize mode")
+            columns_list = [columns] if isinstance(columns, str) else columns
+            result = perform_harmonize(polars_df, columns_list, strategy)
+        
+        elif mode == 'knn':
+            if columns is None:
+                raise ValueError("columns parameter required for knn mode")
+            columns_list = [columns] if isinstance(columns, str) else columns
+            result = perform_knn_imputation(polars_df, columns_list, strategy)
+        
+        else:
+            raise ValueError(
+                f"Unknown mode: {mode}. "
+                f"Valid modes: transpose, onehotencoding, extract, datetime, harmonize, knn"
+            )
+        
+        # Convert back to original backend
+        result = from_polars(result, backend)
+        
+        # Cleanup
+        memory_manager.cleanup()
+        
+        # Wrap result
+        logger.info(f"transform() function complete: {len(result)} rows, {len(result.columns)} columns")
+        return wrap_result(result, 'transform', metadata={'mode': mode})
+    
+    except Exception as e:
+        logger.error(f"Error in transform() function: {str(e)}", error_location="transform")
+        raise
+
+
+__all__ = ['transform']

additory/functions/transform/datetime.py

@@ -0,0 +1,79 @@
+"""
+Normalize datetime columns.
+
+This module provides datetime normalization functionality for the transform function.
+"""
+
+import polars as pl
+from typing import List, Dict, Optional
+
+from additory.common.validation import validate_dataframe, validate_not_empty
+from additory.core.logging import Logger
+
+
+def perform_datetime_normalization(
+    df: pl.DataFrame,
+    columns: List[str],
+    strategy: Optional[Dict] = None
+) -> pl.DataFrame:
+    """
+    Normalize datetime columns.
+    
+    Args:
+        df: Input DataFrame
+        columns: Columns to normalize
+        strategy: Normalization strategy per column
+        
+    Returns:
+        DataFrame with normalized datetime columns
+        
+    Example:
+        >>> result = perform_datetime_normalization(df, 
+        ...     columns=['birth_date'],
+        ...     strategy={
+        ...         'birth_date': {
+        ...             'format': '%Y-%m-%d',
+        ...             'output_format': '%d-%b-%Y'
+        ...         }
+        ...     })
+    """
+    logger = Logger()
+    
+    # Validate
+    validate_dataframe(df)
+    validate_not_empty(df)
+    
+    if not columns:
+        raise ValueError("columns list cannot be empty")
+    
+    # Validate columns exist
+    missing = [col for col in columns if col not in df.columns]
+    if missing:
+        raise ValueError(f"Columns not found: {missing}")
+    
+    logger.info(f"Normalizing {len(columns)} datetime columns")
+    
+    result = df
+    strategy = strategy or {}
+    
+    # Normalize each column
+    for col in columns:
+        col_strategy = strategy.get(col, {})
+        
+        # Parse datetime if string
+        if result[col].dtype == pl.Utf8:
+            fmt = col_strategy.get('format', '%Y-%m-%d')
+            result = result.with_columns(
+                pl.col(col).str.strptime(pl.Datetime, fmt, strict=False).alias(col)
+            )
+        
+        # Convert to desired output format if specified
+        output_fmt = col_strategy.get('output_format')
+        if output_fmt:
+            result = result.with_columns(
+                pl.col(col).dt.strftime(output_fmt).alias(col)
+            )
+    
+    logger.info(f"Datetime normalization complete")
+    
+    return result

additory/functions/transform/extract.py

@@ -0,0 +1,85 @@
+"""
+Extract features from columns.
+
+This module provides feature extraction functionality for the transform function.
+"""
+
+import polars as pl
+from typing import Dict, Optional
+
+from additory.common.validation import validate_dataframe, validate_not_empty
+from additory.common.extractors import (
+    extract_datetime_features,
+    extract_email_features,
+    extract_text_features
+)
+from additory.core.logging import Logger
+
+
+def perform_extract(
+    df: pl.DataFrame,
+    columns: Dict[str, list],
+    strategy: Optional[Dict] = None
+) -> pl.DataFrame:
+    """
+    Extract features from columns.
+    
+    Args:
+        df: Input DataFrame
+        columns: Dictionary mapping column to features to extract
+        strategy: Optional strategy for extraction
+        
+    Returns:
+        DataFrame with extracted features
+        
+    Example:
+        >>> result = perform_extract(df, columns={
+        ...     'timestamp': ['hour', 'day_of_week', 'quarter'],
+        ...     'email': ['domain'],
+        ...     'tags': ['split']
+        ... })
+    """
+    logger = Logger()
+    
+    # Validate
+    validate_dataframe(df)
+    validate_not_empty(df)
+    
+    if not columns:
+        raise ValueError("columns dictionary cannot be empty")
+    
+    logger.info(f"Extracting features from {len(columns)} columns")
+    
+    result = df
+    
+    # Extract features from each column
+    for col, features in columns.items():
+        if col not in df.columns:
+            raise ValueError(f"Column not found: {col}")
+        
+        # Determine column type and extract accordingly
+        col_dtype = df[col].dtype
+        
+        extracted_features = {}
+        
+        if col_dtype in [pl.Datetime, pl.Date]:
+            # Datetime extraction
+            extracted_features = extract_datetime_features(df[col], features)
+        elif col_dtype == pl.Utf8:
+            # Check if it's email or text
+            if 'domain' in features or 'username' in features:
+                extracted_features = extract_email_features(df[col], features)
+            else:
+                extracted_features = extract_text_features(df[col], features)
+        else:
+            logger.warning(f"Unsupported column type for extraction: {col_dtype}")
+            continue
+        
+        # Add extracted features as new columns
+        for feature_name, feature_series in extracted_features.items():
+            new_col_name = f"{col}_{feature_name}"
+            result = result.with_columns(feature_series.alias(new_col_name))
+    
+    logger.info(f"Feature extraction complete: {len(result.columns)} columns")
+    
+    return result

additory/functions/transform/harmonize.py

@@ -0,0 +1,105 @@
+"""
+Harmonize units across columns.
+
+This module provides unit harmonization functionality for the transform function.
+"""
+
+import polars as pl
+from typing import List, Dict, Optional
+
+from additory.common.validation import validate_dataframe, validate_not_empty
+from additory.common.unit_conversions import (
+    convert_weight, convert_temperature, convert_distance,
+    convert_currency, convert_volume, convert_time, detect_unit_type
+)
+from additory.core.logging import Logger
+
+
+def perform_harmonize(
+    df: pl.DataFrame,
+    columns: List[str],
+    strategy: Optional[Dict] = None
+) -> pl.DataFrame:
+    """
+    Harmonize units across columns.
+    
+    Args:
+        df: Input DataFrame
+        columns: Columns to harmonize
+        strategy: Harmonization strategy per column
+        
+    Returns:
+        DataFrame with harmonized units
+        
+    Example:
+        >>> result = perform_harmonize(df,
+        ...     columns=['height', 'weight'],
+        ...     strategy={
+        ...         'height': {'from': 'cm', 'to': 'm'},
+        ...         'weight': {'from': 'lb', 'to': 'kg'}
+        ...     })
+    """
+    logger = Logger()
+    
+    # Validate
+    validate_dataframe(df)
+    validate_not_empty(df)
+    
+    if not columns:
+        raise ValueError("columns list cannot be empty")
+    
+    # Validate columns exist
+    missing = [col for col in columns if col not in df.columns]
+    if missing:
+        raise ValueError(f"Columns not found: {missing}")
+    
+    if not strategy:
+        raise ValueError("strategy is required for harmonization")
+    
+    logger.info(f"Harmonizing {len(columns)} columns")
+    
+    result = df
+    
+    # Harmonize each column
+    for col in columns:
+        if col not in strategy:
+            logger.warning(f"No strategy provided for column: {col}")
+            continue
+        
+        col_strategy = strategy[col]
+        from_unit = col_strategy.get('from')
+        to_unit = col_strategy.get('to')
+        
+        if not from_unit or not to_unit:
+            raise ValueError(f"Both 'from' and 'to' units required for column: {col}")
+        
+        # Detect unit type
+        unit_type = detect_unit_type(from_unit)
+        
+        # Select appropriate conversion function
+        if unit_type == 'weight':
+            converter = convert_weight
+        elif unit_type == 'temperature':
+            converter = convert_temperature
+        elif unit_type == 'distance':
+            converter = convert_distance
+        elif unit_type == 'currency':
+            converter = convert_currency
+        elif unit_type == 'volume':
+            converter = convert_volume
+        elif unit_type == 'time':
+            converter = convert_time
+        else:
+            raise ValueError(f"Unsupported unit type: {unit_type}")
+        
+        # Convert units using map_elements
+        result = result.with_columns(
+            pl.col(col).map_elements(
+                lambda x: converter(x, from_unit, to_unit) if x is not None else None,
+                return_dtype=pl.Float64
+            ).alias(col)
+        )
+    
+    logger.info(f"Unit harmonization complete")
+    
+    return result

additory/functions/transform/knn.py

@@ -0,0 +1,62 @@
+"""
+KNN imputation for missing values.
+
+This module provides KNN imputation functionality for the transform function.
+"""
+
+import polars as pl
+from typing import List, Optional, Dict
+
+from additory.common.validation import validate_dataframe, validate_not_empty
+from additory.common.knn_imputation import knn_impute
+from additory.core.logging import Logger
+
+
+def perform_knn_imputation(
+    df: pl.DataFrame,
+    columns: List[str],
+    strategy: Optional[Dict] = None
+) -> pl.DataFrame:
+    """
+    Impute missing values using KNN.
+    
+    Args:
+        df: Input DataFrame
+        columns: Columns to impute
+        strategy: Imputation strategy (k, weights, etc.)
+        
+    Returns:
+        DataFrame with imputed values
+        
+    Example:
+        >>> result = perform_knn_imputation(df,
+        ...     columns=['age', 'income'],
+        ...     strategy={'k': 5, 'weights': 'distance'})
+    """
+    logger = Logger()
+    
+    # Validate
+    validate_dataframe(df)
+    validate_not_empty(df)
+    
+    if not columns:
+        raise ValueError("columns list cannot be empty")
+    
+    # Validate columns exist
+    missing = [col for col in columns if col not in df.columns]
+    if missing:
+        raise ValueError(f"Columns not found: {missing}")
+    
+    logger.info(f"Performing KNN imputation on {len(columns)} columns")
+    
+    strategy = strategy or {}
+    k = strategy.get('k', 5)
+    weights = strategy.get('weights', 'uniform')
+    metric = strategy.get('metric', 'euclidean')
+    
+    # Perform KNN imputation
+    result = knn_impute(df, columns, k=k, weights=weights, metric=metric)
+    
+    logger.info(f"KNN imputation complete")
+    
+    return result