additory 0.1.0a4__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/common/extractors.py

@@ -0,0 +1,313 @@
+"""
+Feature extraction utilities for Additory.
+
+Provides functions to extract features from various column types
+(datetime, email, text, URL, phone, etc.).
+"""
+
+import re
+from typing import Dict, List, Optional
+import polars as pl
+
+
+def extract_datetime_features(series: pl.Series, features: List[str]) -> Dict[str, pl.Series]:
+    """
+    Extract datetime features from a datetime column.
+    
+    Args:
+        series: Polars Series with datetime values
+        features: List of features to extract
+        
+    Returns:
+        Dictionary mapping feature name to Series
+        
+    Supported Features:
+        - 'year', 'month', 'day', 'hour', 'minute', 'second'
+        - 'day_of_week', 'day_of_year', 'week', 'quarter'
+        - 'time_of_day' ('morning', 'afternoon', 'evening', 'night')
+        - 'is_weekend', 'is_business_day'
+        
+    Example:
+        features = extract_datetime_features(df['timestamp'], ['hour', 'day_of_week'])
+    """
+    result = {}
+    
+    for feature in features:
+        if feature == 'year':
+            result[feature] = series.dt.year()
+        elif feature == 'month':
+            result[feature] = series.dt.month()
+        elif feature == 'day':
+            result[feature] = series.dt.day()
+        elif feature == 'hour':
+            result[feature] = series.dt.hour()
+        elif feature == 'minute':
+            result[feature] = series.dt.minute()
+        elif feature == 'second':
+            result[feature] = series.dt.second()
+        elif feature == 'day_of_week':
+            result[feature] = series.dt.weekday()
+        elif feature == 'day_of_year':
+            result[feature] = series.dt.ordinal_day()
+        elif feature == 'week':
+            result[feature] = series.dt.week()
+        elif feature == 'quarter':
+            result[feature] = series.dt.quarter()
+        elif feature == 'time_of_day':
+            hour = series.dt.hour()
+            time_of_day = pl.when(hour < 6).then(pl.lit('night')) \
+                           .when(hour < 12).then(pl.lit('morning')) \
+                           .when(hour < 18).then(pl.lit('afternoon')) \
+                           .otherwise(pl.lit('evening'))
+            # Evaluate the expression to get a Series
+            result[feature] = pl.select(time_of_day.alias('time_of_day')).to_series()
+        elif feature == 'is_weekend':
+            weekday = series.dt.weekday()
+            result[feature] = (weekday >= 6)
+        elif feature == 'is_business_day':
+            weekday = series.dt.weekday()
+            result[feature] = (weekday < 5)
+        else:
+            raise ValueError(f"Unsupported datetime feature: {feature}")
+    
+    return result
+
+
+def extract_email_features(series: pl.Series, features: List[str]) -> Dict[str, pl.Series]:
+    """
+    Extract features from email addresses.
+    
+    Args:
+        series: Polars Series with email strings
+        features: List of features to extract
+        
+    Returns:
+        Dictionary mapping feature name to Series
+        
+    Supported Features:
+        - 'domain' - Email domain (e.g., 'gmail.com')
+        - 'username' - Username part (before @)
+        - 'tld' - Top-level domain (e.g., 'com')
+        - 'is_free_email' - Boolean (gmail, yahoo, etc.)
+        - 'is_corporate' - Boolean (not free email)
+        
+    Example:
+        features = extract_email_features(df['email'], ['domain', 'is_free_email'])
+    """
+    result = {}
+    
+    # Common free email providers
+    free_providers = {'gmail.com', 'yahoo.com', 'hotmail.com', 'outlook.com', 
+                     'aol.com', 'icloud.com', 'mail.com', 'protonmail.com'}
+    
+    for feature in features:
+        if feature == 'domain':
+            # Extract domain (part after @)
+            result[feature] = series.str.extract(r'@(.+)$', 1)
+        elif feature == 'username':
+            # Extract username (part before @)
+            result[feature] = series.str.extract(r'^([^@]+)@', 1)
+        elif feature == 'tld':
+            # Extract top-level domain (last part after .)
+            result[feature] = series.str.extract(r'\.([^.]+)$', 1)
+        elif feature == 'is_free_email':
+            # Check if domain is in free providers list
+            domain = series.str.extract(r'@(.+)$', 1)
+            result[feature] = domain.is_in(list(free_providers))
+        elif feature == 'is_corporate':
+            # Opposite of is_free_email
+            domain = series.str.extract(r'@(.+)$', 1)
+            result[feature] = ~domain.is_in(list(free_providers))
+        else:
+            raise ValueError(f"Unsupported email feature: {feature}")
+    
+    return result
+
+
+def extract_text_features(series: pl.Series, features: List[str]) -> Dict[str, pl.Series]:
+    """
+    Extract features from text columns.
+    
+    Args:
+        series: Polars Series with text strings
+        features: List of features to extract
+        
+    Returns:
+        Dictionary mapping feature name to Series
+        
+    Supported Features:
+        - 'length' - Character count
+        - 'word_count' - Number of words
+        - 'sentence_count' - Number of sentences
+        - 'has_numbers' - Boolean
+        - 'has_special_chars' - Boolean
+        - 'is_uppercase' - Boolean
+        - 'is_lowercase' - Boolean
+        
+    Example:
+        features = extract_text_features(df['description'], ['length', 'word_count'])
+    """
+    result = {}
+    
+    for feature in features:
+        if feature == 'length':
+            result[feature] = series.str.len_chars()
+        elif feature == 'word_count':
+            # Count words (split by whitespace)
+            result[feature] = series.str.split(' ').list.len()
+        elif feature == 'sentence_count':
+            # Count sentences (split by . ! ?)
+            result[feature] = series.str.count_matches(r'[.!?]')
+        elif feature == 'has_numbers':
+            result[feature] = series.str.contains(r'\d')
+        elif feature == 'has_special_chars':
+            result[feature] = series.str.contains(r'[^a-zA-Z0-9\s]')
+        elif feature == 'is_uppercase':
+            result[feature] = series.str.to_uppercase() == series
+        elif feature == 'is_lowercase':
+            result[feature] = series.str.to_lowercase() == series
+        else:
+            raise ValueError(f"Unsupported text feature: {feature}")
+    
+    return result
+
+
+def split_column(series: pl.Series, delimiter: str, max_splits: Optional[int] = None) -> List[pl.Series]:
+    """
+    Split column by delimiter into multiple columns.
+    
+    Args:
+        series: Polars Series to split
+        delimiter: Delimiter string
+        max_splits: Maximum number of splits (None = unlimited)
+        
+    Returns:
+        List of Series (one per split part)
+        
+    Example:
+        # Split 'red,green,blue' into 3 columns
+        parts = split_column(df['tags'], delimiter=',', max_splits=3)
+    """
+    # Split the series
+    split_series = series.str.split(delimiter)
+    
+    # Determine number of parts
+    if max_splits is None:
+        # Find maximum number of parts
+        max_len = split_series.list.len().max()
+        if max_len is None:
+            # Empty series
+            return []
+        max_parts = int(max_len)
+    else:
+        max_parts = max_splits
+    
+    # Extract each part as a separate series
+    result = []
+    for i in range(max_parts):
+        try:
+            part = split_series.list.get(i, null_on_oob=True)
+            result.append(part)
+        except Exception:
+            # If index is out of bounds, append None series
+            result.append(pl.Series([None] * len(series)))
+    
+    return result
+
+
+def extract_url_features(series: pl.Series, features: List[str]) -> Dict[str, pl.Series]:
+    """
+    Extract features from URLs.
+    
+    Args:
+        series: Polars Series with URL strings
+        features: List of features to extract
+        
+    Returns:
+        Dictionary mapping feature name to Series
+        
+    Supported Features:
+        - 'protocol' - http, https, ftp, etc.
+        - 'domain' - Domain name
+        - 'path' - URL path
+        - 'is_secure' - Boolean (https)
+        
+    Example:
+        features = extract_url_features(df['url'], ['protocol', 'domain'])
+    """
+    result = {}
+    
+    for feature in features:
+        if feature == 'protocol':
+            # Extract protocol (e.g., http, https)
+            result[feature] = series.str.extract(r'^([a-z]+)://', 1)
+        elif feature == 'domain':
+            # Extract domain (between :// and /)
+            result[feature] = series.str.extract(r'://([^/]+)', 1)
+        elif feature == 'path':
+            # Extract path (after domain)
+            result[feature] = series.str.extract(r'://[^/]+(/[^?#]*)', 1)
+        elif feature == 'is_secure':
+            # Check if protocol is https
+            protocol = series.str.extract(r'^([a-z]+)://', 1)
+            result[feature] = (protocol == 'https')
+        else:
+            raise ValueError(f"Unsupported URL feature: {feature}")
+    
+    return result
+
+
+def extract_phone_features(series: pl.Series, features: List[str]) -> Dict[str, pl.Series]:
+    """
+    Extract features from phone numbers.
+    
+    Args:
+        series: Polars Series with phone number strings
+        features: List of features to extract
+        
+    Returns:
+        Dictionary mapping feature name to Series
+        
+    Supported Features:
+        - 'country_code' - Country code
+        - 'area_code' - Area code
+        - 'is_valid' - Boolean (basic validation)
+        
+    Example:
+        features = extract_phone_features(df['phone'], ['country_code', 'area_code'])
+    """
+    result = {}
+    
+    for feature in features:
+        if feature == 'country_code':
+            # Extract country code (e.g., +1, +44)
+            result[feature] = series.str.extract(r'^\+?(\d{1,3})', 1)
+        elif feature == 'area_code':
+            # Extract area code (3 digits after country code)
+            result[feature] = series.str.extract(r'\(?(\d{3})\)?', 1)
+        elif feature == 'is_valid':
+            # Basic validation: has at least 10 digits
+            digits_only = series.str.replace_all(r'\D', '')
+            result[feature] = digits_only.str.len_chars() >= 10
+        else:
+            raise ValueError(f"Unsupported phone feature: {feature}")
+    
+    return result
+
+
+def parse_pattern(series: pl.Series, pattern: str) -> pl.Series:
+    """
+    Extract pattern from strings using regex.
+    
+    Args:
+        series: Polars Series with strings
+        pattern: Regex pattern to extract
+        
+    Returns:
+        Series with extracted values
+        
+    Example:
+        # Extract numbers from strings
+        numbers = parse_pattern(df['text'], r'\\d+')
+    """
+    return series.str.extract(pattern, 0)

additory/common/knn_imputation.py

@@ -0,0 +1,332 @@
+"""
+K-Nearest Neighbors imputation utilities for Additory.
+
+Provides KNN-based imputation for missing values in numeric columns.
+"""
+
+import math
+from typing import List, Optional
+import polars as pl
+import numpy as np
+
+
+def knn_impute(df: pl.DataFrame, columns: List[str], k: int = 5, 
+               weights: str = 'distance', metric: str = 'euclidean') -> pl.DataFrame:
+    """
+    Impute missing values using K-Nearest Neighbors.
+    
+    Args:
+        df: DataFrame with missing values
+        columns: Columns to impute
+        k: Number of neighbors (default: 5)
+        weights: Weighting strategy ('uniform' or 'distance')
+        metric: Distance metric ('euclidean', 'manhattan', 'cosine')
+        
+    Returns:
+        DataFrame with imputed values
+        
+    Example:
+        imputed_df = knn_impute(df, columns=['age', 'income'], k=5, weights='distance')
+    """
+    # Validate parameters
+    validate_knn_parameters(df, columns, k)
+    
+    # Create a copy to avoid modifying original
+    result_df = df.clone()
+    
+    # Get all numeric columns for distance calculation
+    all_numeric_cols = [col for col in df.columns 
+                       if df[col].dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                                           pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                                           pl.Float32, pl.Float64]]
+    
+    # Convert to numpy for easier manipulation
+    data = df.select(columns).to_numpy().copy()  # Make a copy we can modify
+    
+    # For each row with missing values
+    for row_idx in range(len(df)):
+        row = data[row_idx]
+        
+        # Check if this row has any missing values
+        if not np.any(np.isnan(row)):
+            continue
+        
+        # Check if this row has at least some non-missing values in ANY numeric column
+        # (needed for distance calculation)
+        row_all_cols = df.select(all_numeric_cols).row(row_idx)
+        has_any_non_missing = any(val is not None and not (isinstance(val, float) and np.isnan(val)) 
+                                  for val in row_all_cols)
+        if not has_any_non_missing:
+            # All numeric values missing, skip this row
+            continue
+        
+        # Calculate distances to all other rows using ALL numeric columns
+        distances = calculate_distances(df, row_idx, all_numeric_cols, metric)
+        
+        # Find k nearest neighbors
+        neighbor_indices = find_k_nearest(distances, k, exclude_idx=row_idx)
+        
+        # Impute each missing value
+        for col_idx, col_name in enumerate(columns):
+            if np.isnan(row[col_idx]):
+                # Get values from neighbors
+                neighbor_values = []
+                neighbor_distances = []
+                
+                for neighbor_idx in neighbor_indices:
+                    neighbor_value = data[neighbor_idx, col_idx]
+                    if not np.isnan(neighbor_value):
+                        neighbor_values.append(neighbor_value)
+                        neighbor_distances.append(distances[neighbor_idx])
+                
+                # If we have neighbor values, compute weighted average
+                if neighbor_values:
+                    neighbor_values_series = pl.Series(neighbor_values)
+                    neighbor_distances_series = pl.Series(neighbor_distances)
+                    
+                    imputed_value = compute_weighted_average(
+                        neighbor_values_series,
+                        neighbor_distances_series,
+                        weights
+                    )
+                    
+                    # Update the data array
+                    data[row_idx, col_idx] = imputed_value
+    
+    # Replace the columns in the result DataFrame with imputed data
+    for col_idx, col_name in enumerate(columns):
+        result_df = result_df.with_columns(
+            pl.Series(col_name, data[:, col_idx])
+        )
+    
+    return result_df
+
+
+def calculate_distances(df: pl.DataFrame, row_idx: int, columns: List[str], 
+                       metric: str) -> pl.Series:
+    """
+    Calculate distances from a row to all other rows.
+    
+    Args:
+        df: DataFrame
+        row_idx: Index of row to calculate distances from
+        columns: Columns to use for distance calculation
+        metric: Distance metric
+        
+    Returns:
+        Series of distances
+    """
+    data = df.select(columns).to_numpy()
+    row = data[row_idx]
+    
+    distances = []
+    for i in range(len(df)):
+        if i == row_idx:
+            distances.append(float('inf'))  # Exclude self
+        else:
+            other_row = data[i]
+            
+            # Only use non-missing values for distance calculation
+            mask = ~(np.isnan(row) | np.isnan(other_row))
+            
+            if not np.any(mask):
+                # No common non-missing values
+                distances.append(float('inf'))
+            else:
+                row_clean = row[mask]
+                other_clean = other_row[mask]
+                
+                if metric == 'euclidean':
+                    dist = euclidean_distance(
+                        pl.Series(row_clean),
+                        pl.Series(other_clean)
+                    )
+                elif metric == 'manhattan':
+                    dist = manhattan_distance(
+                        pl.Series(row_clean),
+                        pl.Series(other_clean)
+                    )
+                elif metric == 'cosine':
+                    dist = cosine_distance(
+                        pl.Series(row_clean),
+                        pl.Series(other_clean)
+                    )
+                else:
+                    raise ValueError(f"Unsupported metric: {metric}")
+                
+                distances.append(dist)
+    
+    return pl.Series(distances)
+
+
+def find_k_nearest(distances: pl.Series, k: int, exclude_idx: Optional[int] = None) -> List[int]:
+    """
+    Find indices of k nearest neighbors.
+    
+    Args:
+        distances: Series of distances
+        k: Number of neighbors to find
+        exclude_idx: Index to exclude (the row itself)
+        
+    Returns:
+        List of k nearest neighbor indices
+    """
+    # Convert to numpy for easier manipulation
+    dist_array = distances.to_numpy()
+    
+    # Get indices sorted by distance
+    sorted_indices = np.argsort(dist_array)
+    
+    # Filter out infinite distances and excluded index
+    valid_indices = []
+    for idx in sorted_indices:
+        if not np.isinf(dist_array[idx]):
+            if exclude_idx is None or idx != exclude_idx:
+                valid_indices.append(int(idx))
+                if len(valid_indices) >= k:
+                    break
+    
+    return valid_indices
+
+
+def compute_weighted_average(values: pl.Series, distances: pl.Series, weights: str) -> float:
+    """
+    Compute weighted average of neighbor values.
+    
+    Args:
+        values: Values from neighbors
+        distances: Distances to neighbors
+        weights: Weighting strategy ('uniform' or 'distance')
+        
+    Returns:
+        Weighted average value
+    """
+    if weights == 'uniform':
+        # Simple average
+        return float(values.mean())
+    
+    elif weights == 'distance':
+        # Inverse distance weighting
+        values_array = values.to_numpy()
+        distances_array = distances.to_numpy()
+        
+        # Avoid division by zero for very small distances
+        distances_array = np.maximum(distances_array, 1e-10)
+        
+        # Inverse distance weights
+        inv_distances = 1.0 / distances_array
+        
+        # Weighted average
+        weighted_sum = np.sum(values_array * inv_distances)
+        weight_sum = np.sum(inv_distances)
+        
+        return float(weighted_sum / weight_sum)
+    
+    else:
+        raise ValueError(f"Unsupported weighting strategy: {weights}")
+
+
+def euclidean_distance(row1: pl.Series, row2: pl.Series) -> float:
+    """
+    Calculate Euclidean distance between two rows.
+    
+    Args:
+        row1: First row values
+        row2: Second row values
+        
+    Returns:
+        Euclidean distance
+    """
+    diff = row1.to_numpy() - row2.to_numpy()
+    return float(math.sqrt(np.sum(diff ** 2)))
+
+
+def manhattan_distance(row1: pl.Series, row2: pl.Series) -> float:
+    """
+    Calculate Manhattan distance between two rows.
+    
+    Args:
+        row1: First row values
+        row2: Second row values
+        
+    Returns:
+        Manhattan distance
+    """
+    diff = np.abs(row1.to_numpy() - row2.to_numpy())
+    return float(np.sum(diff))
+
+
+def cosine_distance(row1: pl.Series, row2: pl.Series) -> float:
+    """
+    Calculate Cosine distance between two rows.
+    
+    Args:
+        row1: First row values
+        row2: Second row values
+        
+    Returns:
+        Cosine distance
+    """
+    vec1 = row1.to_numpy()
+    vec2 = row2.to_numpy()
+    
+    # Calculate dot product and norms
+    dot_product = np.dot(vec1, vec2)
+    norm1 = np.linalg.norm(vec1)
+    norm2 = np.linalg.norm(vec2)
+    
+    # Avoid division by zero
+    if norm1 == 0 or norm2 == 0:
+        return 1.0  # Maximum distance
+    
+    # Cosine similarity
+    cosine_sim = dot_product / (norm1 * norm2)
+    
+    # Cosine distance (1 - similarity)
+    return float(1.0 - cosine_sim)
+
+
+def validate_knn_parameters(df: pl.DataFrame, columns: List[str], k: int) -> bool:
+    """
+    Validate KNN imputation parameters.
+    
+    Args:
+        df: DataFrame
+        columns: Columns to impute
+        k: Number of neighbors
+        
+    Returns:
+        True if valid
+        
+    Raises:
+        ValueError: If parameters are invalid
+    """
+    # Check columns exist
+    for col in columns:
+        if col not in df.columns:
+            raise ValueError(f"Column '{col}' not found in DataFrame")
+    
+    # Check at least some non-missing values exist (before checking dtype)
+    # This is important because a column with all nulls has dtype Null
+    for col in columns:
+        non_null_count = df[col].null_count()
+        if non_null_count == len(df):
+            raise ValueError(f"Column '{col}' has all missing values, cannot impute")
+    
+    # Check columns are numeric
+    for col in columns:
+        dtype = df[col].dtype
+        if dtype not in [pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                        pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                        pl.Float32, pl.Float64]:
+            raise ValueError(f"Column '{col}' must be numeric, got {dtype}")
+    
+    # Check k is positive
+    if k <= 0:
+        raise ValueError(f"k must be positive, got {k}")
+    
+    # Check k is less than number of rows
+    if k >= len(df):
+        raise ValueError(f"k ({k}) must be less than number of rows ({len(df)})")
+    
+    return True