additory 0.1.0a3__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/utilities/__init__.py

@@ -1,53 +0,0 @@
-# additory/utilities/__init__.py
-# Utilities system - Direct operations (non-.add driven)
-
-"""
-Utilities System Module
-
-This module handles direct operations that don't require .add files:
-- Lookup operations (add.to)
-- Unit conversion (add.harmonize_units)
-- Global settings management
-- Enhanced matchers and resolvers
-- Input validation
-"""
-
-# Utility functionality
-from .lookup import to, fuzzy_lookup, aggregate_lookup
-from .units import harmonize_units, get_supported_units, get_conversion_stats
-from .encoding import onehotencoding
-from .games import play, tictactoe, sudoku
-from .settings import (
-    set_global_settings, get_global_settings, get_setting,
-    set_my_expressions_path, set_my_schemas_path,
-    get_my_expressions_path, get_my_schemas_path,
-    set_backend, set_precision, enable_cache, disable_cache
-)
-from .validators import (
-    validate_dataframe, validate_columns_exist, validate_numeric_column,
-    validate_string_column, is_dataframe
-)
-
-__all__ = [
-    # Lookup functionality
-    'to', 'fuzzy_lookup', 'aggregate_lookup',
-    
-    # Unit conversion
-    'harmonize_units', 'get_supported_units', 'get_conversion_stats',
-    
-    # Encoding
-    'onehotencoding',
-    
-    # Games (Easter egg)
-    'play', 'tictactoe', 'sudoku',
-    
-    # Settings management
-    'set_global_settings', 'get_global_settings', 'get_setting',
-    'set_my_expressions_path', 'set_my_schemas_path',
-    'get_my_expressions_path', 'get_my_schemas_path',
-    'set_backend', 'set_precision', 'enable_cache', 'disable_cache',
-    
-    # Validation
-    'validate_dataframe', 'validate_columns_exist', 'validate_numeric_column',
-    'validate_string_column', 'is_dataframe'
-]

additory/utilities/encoding.py

@@ -1,600 +0,0 @@
-# additory/utilities/encoding.py
-# One-hot encoding and other encoding utilities
-
-"""
-Encoding Utilities Module
-
-Provides encoding operations for categorical data:
-- One-hot encoding with native backend support
-- Smart cardinality handling
-- Column name conflict resolution
-"""
-
-from typing import Union, Optional, List, Dict, Any, Tuple
-from collections import Counter
-import warnings
-
-# Import from harmonized common module
-from additory.common import (
-    detect_backend,
-    validate_dataframe,
-    validate_columns_exist,
-    validate_positive_number,
-    validate_ratio,
-    validate_integer_in_range,
-    ValidationError,
-    EncodingError,
-    sanitize_column_name,
-    generate_safe_column_name,
-    BackendType
-)
-
-# Import column positioning
-from additory.core.column_positioning import position_columns
-
-# Backend imports
-import pandas as pd
-import numpy as np
-
-try:
-    import polars as pl
-    HAS_POLARS = True
-except ImportError:
-    HAS_POLARS = False
-    pl = None
-
-try:
-    import cudf
-    HAS_CUDF = True
-except ImportError:
-    HAS_CUDF = False
-    cudf = None
-
-
-DataFrame = Union[pd.DataFrame, 'pl.DataFrame', 'cudf.DataFrame']
-
-
-# Constants
-_MAX_CATEGORIES_ABSOLUTE = 200
-
-
-def onehotencoding(
-    df: DataFrame,
-    column: str,
-    *,
-    max_categories: int = 50,
-    max_cardinality_ratio: float = 0.5,
-    handle_overflow: str = "error",
-    position: str = "after",
-    drop_original: bool = True,
-    prefix: Optional[str] = None,
-    suffix: Optional[str] = None,
-    check_id_column: bool = True,
-    max_column_name_length: int = 63
-) -> DataFrame:
-    """
-    One-hot encode a categorical column into binary uint8 columns.
-    
-    Validates all column names before any data processing to fail fast.
-    Uses native backend operations (no conversions) for optimal performance.
-    
-    Parameters
-    ----------
-    df : DataFrame
-        Input dataframe (pandas, polars, or cudf)
-    column : str
-        Column name to encode
-    max_categories : int, default 50
-        Maximum number of categories to encode (includes "other" if used).
-        Cannot exceed 200 (hard limit).
-    max_cardinality_ratio : float, default 0.5
-        Maximum ratio of unique values to total rows (0.0-1.0).
-        If exceeded, raises error unless handle_overflow="warn".
-        Set to 1.0 to disable check.
-    handle_overflow : str, default "error"
-        How to handle when unique values exceed max_categories:
-        - "error": Raise EncodingError (default)
-        - "top_n": Keep top N most frequent, group rest as "other"
-        - "top_n:N": Keep top N most frequent + "other" (e.g., "top_n:20")
-        - "warn": Proceed with warning (if cardinality ratio allows)
-    position : str, default "after"
-        Where to insert new columns: "after", "before", "end", "start"
-    drop_original : bool, default True
-        Whether to drop the original column after encoding
-    prefix : str, optional
-        Prefix for new column names (max 20 chars)
-    suffix : str, optional
-        Suffix for new column names (max 20 chars)
-    check_id_column : bool, default True
-        Check if column appears to be an ID column and raise error
-    max_column_name_length : int, default 63
-        Maximum length for generated column names (SQL compatibility)
-        
-    Returns
-    -------
-    DataFrame
-        DataFrame with one-hot encoded uint8 columns (0 or 1)
-        
-    Raises
-    ------
-    ValidationError
-        - If column does not exist
-        - If parameters are invalid
-    EncodingError
-        - If cardinality exceeds max_categories and handle_overflow="error"
-        - If unique ratio exceeds max_cardinality_ratio
-        - If generated column names have duplicates (after truncation)
-        - If generated column names conflict with existing columns
-        - If max_categories exceeds hard limit (200)
-        - If column appears to be an ID column and check_id_column=True
-        
-    Examples
-    --------
-    Basic usage:
-    >>> df = pl.DataFrame({"color": ["red", "blue", "red", "green"]})
-    >>> result = add.onehotencoding(df, "color")
-    # Creates: color_blue, color_green, color_red (uint8, sorted)
-    
-    With prefix to avoid conflicts:
-    >>> result = add.onehotencoding(df, "color", prefix="ohe", drop_original=False)
-    # Creates: ohe_color_blue, ohe_color_green, ohe_color_red
-    # Keeps: color
-    
-    Handle high cardinality with top_n:
-    >>> result = add.onehotencoding(
-    ...     df, "category", 
-    ...     max_categories=10, 
-    ...     handle_overflow="top_n:9"
-    ... )
-    # Creates: top 9 categories + category_other
-    
-    Allow high cardinality:
-    >>> result = add.onehotencoding(
-    ...     df, "user_id", 
-    ...     max_cardinality_ratio=1.0,
-    ...     max_categories=100,
-    ...     check_id_column=False
-    ... )
-    # Proceeds even if user_id looks like an ID column
-    
-    Notes
-    -----
-    - All validation happens before data processing (fail-fast)
-    - Generated column names are checked for duplicates and conflicts
-    - Category values are sorted alphabetically in output columns
-    - "other" category (if used) always appears last
-    - Uses uint8 dtype (0 or 1) for memory efficiency
-    - Native backend operations (no conversions) for performance
-    """
-    
-    # ============================================
-    # PHASE 1: VALIDATION (No data processing)
-    # ============================================
-    
-    # Detect backend
-    backend = detect_backend(df)
-    
-    # Validate parameters
-    _validate_parameters(
-        max_categories, max_cardinality_ratio, handle_overflow,
-        position, prefix, suffix, max_column_name_length
-    )
-    
-    # Parse handle_overflow
-    overflow_mode, top_n = _parse_handle_overflow(handle_overflow, max_categories)
-    
-    # Validate dataframe and column
-    validate_dataframe(df, "input dataframe")
-    validate_columns_exist(df, column, "input dataframe")
-    
-    # Get column statistics (NATIVE operations)
-    stats = _get_column_stats_native(df, column, backend)
-    
-    # Check if ID column
-    if check_id_column:
-        _check_id_column(column, stats)
-    
-    # Check cardinality ratio
-    if max_cardinality_ratio < 1.0:
-        _check_cardinality_ratio(column, stats, max_cardinality_ratio)
-    
-    # Determine categories to encode
-    categories = _determine_categories(
-        df, column, backend, stats['n_unique'], 
-        max_categories, overflow_mode, top_n
-    )
-    
-    # Generate column names
-    generated_names = _generate_column_names(
-        column, categories, prefix, suffix, max_column_name_length
-    )
-    
-    # Validate column names (duplicates and conflicts)
-    _validate_column_names(generated_names, df, column, drop_original)
-    
-    # ✅ All validations passed!
-    
-    # ============================================
-    # PHASE 2: EXECUTION (Data processing)
-    # ============================================
-    
-    # Create encoded columns (NATIVE operations)
-    new_columns_dict = _create_encoded_columns_native(
-        df, column, categories, generated_names, backend
-    )
-    
-    # Add new columns to dataframe (NATIVE operations)
-    result = _add_columns_native(df, new_columns_dict, backend)
-    
-    # Position columns
-    if position != "end":
-        # Convert position to column_positioning format
-        if position == "after":
-            position_spec = f"after:{column}" if column in result.columns else "end"
-        elif position == "before":
-            position_spec = f"before:{column}" if column in result.columns else "start"
-        else:
-            position_spec = position
-        
-        # Column positioning works with pandas
-        if backend == 'pandas':
-            result = position_columns(result, generated_names, position_spec)
-        else:
-            # For polars/cudf, do manual positioning
-            result = _position_columns_native(
-                result, generated_names, position, column, backend
-            )
-    
-    # Drop original if requested
-    if drop_original:
-        result = _drop_column_native(result, column, backend)
-    
-    return result
-
-
-def _validate_parameters(max_categories, max_cardinality_ratio, handle_overflow,
-                        position, prefix, suffix, max_column_name_length):
-    """Validate all parameters."""
-    # Validate max_categories
-    validate_integer_in_range(
-        max_categories, "max_categories", 
-        min_val=1, max_val=_MAX_CATEGORIES_ABSOLUTE
-    )
-    
-    # Validate max_cardinality_ratio
-    validate_ratio(max_cardinality_ratio, "max_cardinality_ratio")
-    
-    # Validate handle_overflow format
-    if not isinstance(handle_overflow, str):
-        raise ValidationError("handle_overflow must be a string")
-    
-    # Validate position
-    valid_positions = ["after", "before", "end", "start"]
-    if position not in valid_positions:
-        raise ValidationError(
-            f"Invalid position: '{position}'. Must be one of: {valid_positions}"
-        )
-    
-    # Validate prefix/suffix length
-    if prefix and len(prefix) > 20:
-        raise ValidationError("prefix must be <= 20 characters")
-    if suffix and len(suffix) > 20:
-        raise ValidationError("suffix must be <= 20 characters")
-    
-    # Validate max_column_name_length
-    if max_column_name_length < 10:
-        raise ValidationError("max_column_name_length must be >= 10")
-
-
-def _parse_handle_overflow(handle_overflow: str, max_categories: int) -> Tuple[str, Optional[int]]:
-    """
-    Parse handle_overflow parameter.
-    
-    Returns:
-        Tuple of (mode, top_n_count)
-    """
-    if ":" in handle_overflow:
-        mode, value = handle_overflow.split(":", 1)
-        mode = mode.strip()
-        value = value.strip()
-        
-        if mode != "top_n":
-            raise ValidationError(
-                f"Invalid handle_overflow format: '{handle_overflow}'. "
-                f"Only 'top_n' supports ':' syntax."
-            )
-        
-        if value == "auto":
-            top_n = max_categories - 1
-        else:
-            try:
-                top_n = int(value)
-            except ValueError:
-                raise ValidationError(
-                    f"Invalid top_n value: '{value}'. Must be integer or 'auto'."
-                )
-            
-            if top_n >= max_categories:
-                raise ValidationError(
-                    f"top_n ({top_n}) must be less than max_categories ({max_categories})"
-                )
-            if top_n < 1:
-                raise ValidationError(f"top_n must be at least 1, got {top_n}")
-        
-        return mode, top_n
-    
-    else:
-        mode = handle_overflow.strip()
-        if mode == "top_n":
-            # Default: leave room for "other"
-            top_n = max_categories - 1
-            return mode, top_n
-        elif mode in ["error", "warn"]:
-            return mode, None
-        else:
-            raise ValidationError(
-                f"Invalid handle_overflow: '{handle_overflow}'. "
-                f"Must be 'error', 'warn', 'top_n', or 'top_n:N'"
-            )
-
-
-def _get_column_stats_native(df, column: str, backend: BackendType) -> Dict[str, Any]:
-    """Get column statistics using native backend operations."""
-    if backend == 'polars':
-        n_rows = df.height
-        n_unique = df[column].n_unique()
-        unique_values = df[column].unique().to_list()
-        
-    elif backend == 'cudf':
-        n_rows = len(df)
-        n_unique = df[column].nunique()
-        unique_values = df[column].unique().to_arrow().to_pylist()
-        
-    elif backend == 'pandas':
-        n_rows = len(df)
-        n_unique = df[column].nunique()
-        unique_values = df[column].unique().tolist()
-    
-    else:
-        raise EncodingError(f"Unsupported backend: {backend}")
-    
-    return {
-        'n_rows': n_rows,
-        'n_unique': n_unique,
-        'unique_values': unique_values,
-        'cardinality_ratio': n_unique / n_rows if n_rows > 0 else 0
-    }
-
-
-def _check_id_column(column: str, stats: Dict[str, Any]):
-    """Check if column appears to be an ID column."""
-    ratio = stats['cardinality_ratio']
-    
-    # Check cardinality ratio
-    if ratio > 0.95:
-        raise EncodingError(
-            f"Column '{column}' appears to be an ID column "
-            f"(cardinality ratio: {ratio:.2%}). "
-            f"Set check_id_column=False to override."
-        )
-    
-    # Check column name patterns
-    col_lower = column.lower()
-    id_patterns = ['_id', 'id_', 'uuid', '_key', 'key_']
-    if any(pattern in col_lower for pattern in id_patterns):
-        if ratio > 0.8:
-            raise EncodingError(
-                f"Column '{column}' appears to be an ID column "
-                f"(name pattern + {ratio:.2%} cardinality). "
-                f"Set check_id_column=False to override."
-            )
-
-
-def _check_cardinality_ratio(column: str, stats: Dict[str, Any], max_ratio: float):
-    """Check cardinality ratio."""
-    ratio = stats['cardinality_ratio']
-    if ratio > max_ratio:
-        raise EncodingError(
-            f"Column '{column}' has high cardinality ratio: {ratio:.2%} "
-            f"(threshold: {max_ratio:.2%}). "
-            f"Set max_cardinality_ratio=1.0 to override."
-        )
-
-
-def _determine_categories(df, column: str, backend: BackendType, n_unique: int,
-                         max_categories: int, overflow_mode: str, 
-                         top_n: Optional[int]) -> List[str]:
-    """Determine which categories to encode."""
-    if n_unique > max_categories:
-        if overflow_mode == "error":
-            raise EncodingError(
-                f"Column '{column}' has {n_unique} unique values, "
-                f"exceeds max_categories={max_categories}. "
-                f"Use handle_overflow='top_n' or increase max_categories."
-            )
-        elif overflow_mode == "top_n":
-            # Get top N most frequent categories
-            categories = _get_top_n_categories_native(df, column, backend, top_n)
-            categories.append("other")  # Add "other" category
-            return categories
-        elif overflow_mode == "warn":
-            warnings.warn(
-                f"Encoding {n_unique} categories (exceeds max_categories={max_categories})"
-            )
-            return _get_all_categories_sorted_native(df, column, backend)
-    else:
-        return _get_all_categories_sorted_native(df, column, backend)
-
-
-def _get_top_n_categories_native(df, column: str, backend: BackendType, 
-                                top_n: int) -> List[str]:
-    """Get top N most frequent categories using native operations."""
-    if backend == 'polars':
-        vc = df[column].value_counts(sort=True).head(top_n)
-        return vc[column].to_list()
-        
-    elif backend == 'cudf':
-        vc = df[column].value_counts().sort_values(ascending=False).head(top_n)
-        return vc.index.to_arrow().to_pylist()
-        
-    elif backend == 'pandas':
-        vc = df[column].value_counts().head(top_n)
-        return vc.index.tolist()
-    
-    else:
-        raise EncodingError(f"Unsupported backend: {backend}")
-
-
-def _get_all_categories_sorted_native(df, column: str, backend: BackendType) -> List[str]:
-    """Get all unique categories sorted."""
-    if backend == 'polars':
-        return sorted(df[column].unique().to_list())
-    elif backend == 'cudf':
-        return sorted(df[column].unique().to_arrow().to_pylist())
-    elif backend == 'pandas':
-        return sorted(df[column].unique().tolist())
-    else:
-        raise EncodingError(f"Unsupported backend: {backend}")
-
-
-def _generate_column_names(column: str, categories: List[str], 
-                          prefix: Optional[str], suffix: Optional[str],
-                          max_length: int) -> List[str]:
-    """Generate column names for encoded categories."""
-    generated_names = []
-    
-    for category in categories:
-        # Build parts
-        parts = []
-        if prefix:
-            parts.append(prefix[:20])  # Limit prefix
-        parts.append(column)
-        parts.append(str(category))
-        if suffix:
-            parts.append(suffix[:20])  # Limit suffix
-        
-        # Join with underscores
-        full_name = "_".join(parts)
-        
-        # Truncate if needed (preserve end for uniqueness)
-        if len(full_name) > max_length:
-            # Keep start and end
-            keep_start = max_length // 2
-            keep_end = max_length - keep_start
-            full_name = full_name[:keep_start] + full_name[-keep_end:]
-        
-        generated_names.append(full_name)
-    
-    return generated_names
-
-
-def _validate_column_names(generated_names: List[str], df, column: str, 
-                          drop_original: bool):
-    """Validate generated column names for duplicates and conflicts."""
-    # Check for duplicates in generated names
-    duplicates = [name for name, count in Counter(generated_names).items() if count > 1]
-    if duplicates:
-        raise EncodingError(
-            f"Column name generation resulted in {len(duplicates)} duplicate names. "
-            f"Examples: {duplicates[:3]}. "
-            f"Try using a shorter prefix/suffix or cleaning category values."
-        )
-    
-    # Check for conflicts with existing columns
-    existing_cols = set(df.columns)
-    if drop_original:
-        existing_cols.discard(column)  # Will be dropped, so not a conflict
-    
-    conflicts = set(generated_names) & existing_cols
-    if conflicts:
-        raise EncodingError(
-            f"Generated column names conflict with existing columns: {list(conflicts)[:5]}. "
-            f"Rename existing columns or use prefix parameter."
-        )
-
-
-def _create_encoded_columns_native(df, column: str, categories: List[str],
-                                  col_names: List[str], backend: BackendType) -> Dict[str, Any]:
-    """Create one-hot encoded columns using native backend operations."""
-    if backend == 'polars':
-        new_cols = {}
-        for category, col_name in zip(categories, col_names):
-            if category == "other":
-                mask = ~df[column].is_in(categories[:-1])
-            else:
-                mask = df[column] == category
-            new_cols[col_name] = mask.cast(pl.UInt8)
-        return new_cols
-        
-    elif backend == 'cudf':
-        new_cols = {}
-        for category, col_name in zip(categories, col_names):
-            if category == "other":
-                mask = ~df[column].isin(categories[:-1])
-            else:
-                mask = df[column] == category
-            new_cols[col_name] = mask.astype('uint8')
-        return new_cols
-        
-    elif backend == 'pandas':
-        new_cols = {}
-        for category, col_name in zip(categories, col_names):
-            if category == "other":
-                mask = ~df[column].isin(categories[:-1])
-            else:
-                mask = df[column] == category
-            new_cols[col_name] = mask.astype(np.uint8)
-        return new_cols
-    
-    else:
-        raise EncodingError(f"Unsupported backend: {backend}")
-
-
-def _add_columns_native(df, new_columns: Dict[str, Any], backend: BackendType):
-    """Add new columns to dataframe using native operations."""
-    if backend == 'polars':
-        return df.with_columns([pl.lit(col_data).alias(col_name) 
-                               for col_name, col_data in new_columns.items()])
-    elif backend == 'cudf' or backend == 'pandas':
-        result = df.copy()
-        for col_name, col_data in new_columns.items():
-            result[col_name] = col_data
-        return result
-    else:
-        raise EncodingError(f"Unsupported backend: {backend}")
-
-
-def _position_columns_native(df, new_columns: List[str], position: str, 
-                            reference_col: str, backend: BackendType):
-    """Position columns using native operations (for polars/cudf)."""
-    all_cols = list(df.columns)
-    existing_cols = [c for c in all_cols if c not in new_columns]
-    
-    if position == "start":
-        new_order = new_columns + existing_cols
-    elif position == "after":
-        if reference_col in existing_cols:
-            idx = existing_cols.index(reference_col) + 1
-            new_order = existing_cols[:idx] + new_columns + existing_cols[idx:]
-        else:
-            new_order = existing_cols + new_columns
-    elif position == "before":
-        if reference_col in existing_cols:
-            idx = existing_cols.index(reference_col)
-            new_order = existing_cols[:idx] + new_columns + existing_cols[idx:]
-        else:
-            new_order = new_columns + existing_cols
-    else:  # "end"
-        new_order = existing_cols + new_columns
-    
-    return df[new_order]
-
-
-def _drop_column_native(df, column: str, backend: BackendType):
-    """Drop column using native operations."""
-    if backend == 'polars':
-        return df.drop(column)
-    elif backend == 'cudf' or backend == 'pandas':
-        return df.drop(columns=[column])
-    else:
-        raise EncodingError(f"Unsupported backend: {backend}")