additory 0.1.0a1__py3-none-any.whl

additory/common/distributions.py

@@ -0,0 +1,737 @@
+"""
+Distribution Strategies for Data Augmentation
+
+Provides statistical distribution-based data generation:
+- Normal (Gaussian) distribution
+- Uniform distribution
+- Skewed distributions (left/right)
+- Custom distributions based on existing data
+"""
+
+from typing import List, Optional, Tuple
+import warnings
+
+import numpy as np
+
+from additory.common.exceptions import ValidationError, AugmentError
+
+
+class DistributionType:
+    """Supported distribution types."""
+    NORMAL = "normal"
+    UNIFORM = "uniform"
+    SKEWED_LEFT = "skewed_left"
+    SKEWED_RIGHT = "skewed_right"
+    BETA = "beta"
+    GAMMA = "gamma"
+    EXPONENTIAL = "exponential"
+    KDE = "kde"
+    AUTO = "auto"
+
+
+def estimate_distribution_params(y: np.ndarray) -> Tuple[float, float, float, float]:
+    """
+    Estimate distribution parameters from data.
+    
+    Args:
+        y: Data values
+        
+    Returns:
+        Tuple of (mean, std, min, max)
+    """
+    return float(np.mean(y)), float(np.std(y)), float(np.min(y)), float(np.max(y))
+
+
+def calculate_skewness(y: np.ndarray) -> float:
+    """
+    Calculate skewness of data.
+    
+    Skewness measures asymmetry of distribution:
+    - 0: Symmetric (normal)
+    - > 0: Right-skewed (tail on right)
+    - < 0: Left-skewed (tail on left)
+    
+    Args:
+        y: Data values
+        
+    Returns:
+        Skewness value
+    """
+    n = len(y)
+    if n < 3:
+        return 0.0
+    
+    mean_y = np.mean(y)
+    std_y = np.std(y)
+    
+    if std_y == 0:
+        return 0.0
+    
+    # Calculate third moment
+    skew = np.sum(((y - mean_y) / std_y) ** 3) / n
+    
+    return float(skew)
+
+
+def detect_distribution_type(y: np.ndarray) -> str:
+    """
+    Detect distribution type from data.
+    
+    Args:
+        y: Data values
+        
+    Returns:
+        Distribution type: normal, skewed_left, skewed_right, or uniform
+    """
+    skewness = calculate_skewness(y)
+    
+    # Check for uniform distribution (low variance relative to range)
+    std_y = np.std(y)
+    range_y = np.max(y) - np.min(y)
+    
+    if range_y > 0:
+        cv = std_y / range_y  # Coefficient of variation relative to range
+        # Uniform distribution has CV ≈ 0.289
+        if 0.25 < cv < 0.35 and abs(skewness) < 0.3:
+            return DistributionType.UNIFORM
+    
+    # Check skewness
+    if abs(skewness) < 0.5:
+        return DistributionType.NORMAL
+    elif skewness > 0.5:
+        return DistributionType.SKEWED_RIGHT
+    else:
+        return DistributionType.SKEWED_LEFT
+
+
+def generate_normal(
+    n_rows: int,
+    mean: Optional[float] = None,
+    std: Optional[float] = None,
+    data: Optional[np.ndarray] = None,
+    seed: Optional[int] = None,
+    clip: bool = True
+) -> List[float]:
+    """
+    Generate values from normal (Gaussian) distribution.
+    
+    Args:
+        n_rows: Number of values to generate
+        mean: Mean of distribution (estimated from data if None)
+        std: Standard deviation (estimated from data if None)
+        data: Existing data to estimate parameters from
+        seed: Random seed for reproducibility
+        clip: Whether to clip values to data range
+        
+    Returns:
+        List of generated values
+        
+    Raises:
+        ValidationError: If neither parameters nor data provided
+    """
+    # Estimate parameters from data if not provided
+    if mean is None or std is None:
+        if data is None:
+            raise ValidationError(
+                "Must provide either (mean, std) or data for normal distribution"
+            )
+        
+        est_mean, est_std, data_min, data_max = estimate_distribution_params(data)
+        
+        if mean is None:
+            mean = est_mean
+        if std is None:
+            std = est_std
+    
+    # Validate parameters
+    if std <= 0:
+        raise ValidationError(f"Standard deviation must be positive, got {std}")
+    
+    # Generate values
+    if seed is not None:
+        np.random.seed(seed)
+    
+    values = np.random.normal(mean, std, n_rows)
+    
+    # Clip to data range if requested
+    if clip and data is not None:
+        data_min = np.min(data)
+        data_max = np.max(data)
+        values = np.clip(values, data_min, data_max)
+    
+    return values.tolist()
+
+
+def generate_uniform(
+    n_rows: int,
+    min_val: Optional[float] = None,
+    max_val: Optional[float] = None,
+    data: Optional[np.ndarray] = None,
+    seed: Optional[int] = None
+) -> List[float]:
+    """
+    Generate values from uniform distribution.
+    
+    Args:
+        n_rows: Number of values to generate
+        min_val: Minimum value (estimated from data if None)
+        max_val: Maximum value (estimated from data if None)
+        data: Existing data to estimate parameters from
+        seed: Random seed for reproducibility
+        
+    Returns:
+        List of generated values
+        
+    Raises:
+        ValidationError: If neither parameters nor data provided
+    """
+    # Estimate parameters from data if not provided
+    if min_val is None or max_val is None:
+        if data is None:
+            raise ValidationError(
+                "Must provide either (min_val, max_val) or data for uniform distribution"
+            )
+        
+        _, _, data_min, data_max = estimate_distribution_params(data)
+        
+        if min_val is None:
+            min_val = data_min
+        if max_val is None:
+            max_val = data_max
+    
+    # Validate parameters
+    if min_val >= max_val:
+        raise ValidationError(
+            f"min_val must be less than max_val, got min={min_val}, max={max_val}"
+        )
+    
+    # Generate values
+    if seed is not None:
+        np.random.seed(seed)
+    
+    values = np.random.uniform(min_val, max_val, n_rows)
+    
+    return values.tolist()
+
+
+def generate_skewed(
+    n_rows: int,
+    direction: str,
+    mean: Optional[float] = None,
+    std: Optional[float] = None,
+    skewness: float = 1.0,
+    data: Optional[np.ndarray] = None,
+    seed: Optional[int] = None,
+    clip: bool = True
+) -> List[float]:
+    """
+    Generate values from skewed distribution.
+    
+    Uses log-normal distribution for right skew and reflected log-normal for left skew.
+    
+    Args:
+        n_rows: Number of values to generate
+        direction: 'left' or 'right'
+        mean: Target mean (estimated from data if None)
+        std: Target standard deviation (estimated from data if None)
+        skewness: Degree of skewness (default: 1.0)
+        data: Existing data to estimate parameters from
+        seed: Random seed for reproducibility
+        clip: Whether to clip values to data range
+        
+    Returns:
+        List of generated values
+        
+    Raises:
+        ValidationError: If parameters invalid
+    """
+    # Validate direction
+    if direction not in ['left', 'right']:
+        raise ValidationError(f"Direction must be 'left' or 'right', got '{direction}'")
+    
+    # Estimate parameters from data if not provided
+    if mean is None or std is None:
+        if data is None:
+            raise ValidationError(
+                "Must provide either (mean, std) or data for skewed distribution"
+            )
+        
+        est_mean, est_std, data_min, data_max = estimate_distribution_params(data)
+        
+        if mean is None:
+            mean = est_mean
+        if std is None:
+            std = est_std
+    
+    # Validate parameters
+    if std <= 0:
+        raise ValidationError(f"Standard deviation must be positive, got {std}")
+    
+    # Generate values
+    if seed is not None:
+        np.random.seed(seed)
+    
+    # Use log-normal distribution for skewness
+    # Adjust parameters to match target mean and std
+    sigma = np.sqrt(np.log(1 + (std / mean) ** 2))
+    mu = np.log(mean) - 0.5 * sigma ** 2
+    
+    # Scale sigma by skewness parameter
+    sigma *= abs(skewness)
+    
+    if direction == 'right':
+        # Right-skewed: log-normal
+        values = np.random.lognormal(mu, sigma, n_rows)
+    else:
+        # Left-skewed: reflected log-normal
+        values = np.random.lognormal(mu, sigma, n_rows)
+        # Reflect around mean
+        values = 2 * mean - values
+    
+    # Clip to data range if requested
+    if clip and data is not None:
+        data_min = np.min(data)
+        data_max = np.max(data)
+        values = np.clip(values, data_min, data_max)
+    
+    return values.tolist()
+
+
+def generate_beta(
+    n_rows: int,
+    alpha: Optional[float] = None,
+    beta_param: Optional[float] = None,
+    data: Optional[np.ndarray] = None,
+    seed: Optional[int] = None,
+    scale_min: float = 0.0,
+    scale_max: float = 1.0
+) -> List[float]:
+    """
+    Generate values from beta distribution.
+    
+    Beta distribution is bounded between 0 and 1 (or scaled range).
+    Useful for percentages, probabilities, proportions.
+    
+    Args:
+        n_rows: Number of values to generate
+        alpha: Shape parameter (> 0)
+        beta_param: Shape parameter (> 0)
+        data: Existing data to estimate parameters from
+        seed: Random seed for reproducibility
+        scale_min: Minimum value for scaling (default: 0)
+        scale_max: Maximum value for scaling (default: 1)
+        
+    Returns:
+        List of generated values
+        
+    Raises:
+        ValidationError: If parameters invalid
+    """
+    # Estimate parameters from data if not provided
+    if alpha is None or beta_param is None:
+        if data is None:
+            raise ValidationError(
+                "Must provide either (alpha, beta) or data for beta distribution"
+            )
+        
+        # Normalize data to [0, 1]
+        data_min = np.min(data)
+        data_max = np.max(data)
+        
+        if data_max == data_min:
+            raise ValidationError("Data has no variance, cannot fit beta distribution")
+        
+        normalized = (data - data_min) / (data_max - data_min)
+        
+        # Method of moments estimation
+        mean = np.mean(normalized)
+        var = np.var(normalized)
+        
+        # Avoid edge cases
+        mean = np.clip(mean, 0.01, 0.99)
+        var = np.clip(var, 0.001, mean * (1 - mean) * 0.99)
+        
+        # Estimate alpha and beta
+        alpha = mean * ((mean * (1 - mean) / var) - 1)
+        beta_param = (1 - mean) * ((mean * (1 - mean) / var) - 1)
+        
+        # Use data range for scaling
+        scale_min = data_min
+        scale_max = data_max
+    
+    # Validate parameters
+    if alpha <= 0 or beta_param <= 0:
+        raise ValidationError(
+            f"Alpha and beta must be positive, got alpha={alpha}, beta={beta_param}"
+        )
+    
+    # Generate values
+    if seed is not None:
+        np.random.seed(seed)
+    
+    values = np.random.beta(alpha, beta_param, n_rows)
+    
+    # Scale to desired range
+    values = values * (scale_max - scale_min) + scale_min
+    
+    return values.tolist()
+
+
+def generate_gamma(
+    n_rows: int,
+    shape: Optional[float] = None,
+    scale: Optional[float] = None,
+    data: Optional[np.ndarray] = None,
+    seed: Optional[int] = None
+) -> List[float]:
+    """
+    Generate values from gamma distribution.
+    
+    Gamma distribution is for positive values, often right-skewed.
+    Useful for waiting times, sizes, amounts.
+    
+    Args:
+        n_rows: Number of values to generate
+        shape: Shape parameter (k, > 0)
+        scale: Scale parameter (theta, > 0)
+        data: Existing data to estimate parameters from
+        seed: Random seed for reproducibility
+        
+    Returns:
+        List of generated values
+        
+    Raises:
+        ValidationError: If parameters invalid
+    """
+    # Estimate parameters from data if not provided
+    if shape is None or scale is None:
+        if data is None:
+            raise ValidationError(
+                "Must provide either (shape, scale) or data for gamma distribution"
+            )
+        
+        # Check for non-positive values
+        if np.any(data <= 0):
+            raise ValidationError(
+                "Gamma distribution requires all positive values"
+            )
+        
+        # Method of moments estimation
+        mean = np.mean(data)
+        var = np.var(data)
+        
+        if var == 0:
+            raise ValidationError("Data has no variance, cannot fit gamma distribution")
+        
+        # shape = mean^2 / var, scale = var / mean
+        shape = (mean ** 2) / var
+        scale = var / mean
+    
+    # Validate parameters
+    if shape <= 0 or scale <= 0:
+        raise ValidationError(
+            f"Shape and scale must be positive, got shape={shape}, scale={scale}"
+        )
+    
+    # Generate values
+    if seed is not None:
+        np.random.seed(seed)
+    
+    values = np.random.gamma(shape, scale, n_rows)
+    
+    return values.tolist()
+
+
+def generate_exponential_dist(
+    n_rows: int,
+    rate: Optional[float] = None,
+    data: Optional[np.ndarray] = None,
+    seed: Optional[int] = None
+) -> List[float]:
+    """
+    Generate values from exponential distribution.
+    
+    Exponential distribution models time between events.
+    Memoryless property. Always positive.
+    
+    Args:
+        n_rows: Number of values to generate
+        rate: Rate parameter (lambda, > 0). Mean = 1/rate
+        data: Existing data to estimate parameters from
+        seed: Random seed for reproducibility
+        
+    Returns:
+        List of generated values
+        
+    Raises:
+        ValidationError: If parameters invalid
+    """
+    # Estimate parameters from data if not provided
+    if rate is None:
+        if data is None:
+            raise ValidationError(
+                "Must provide either rate or data for exponential distribution"
+            )
+        
+        # Check for non-positive values
+        if np.any(data <= 0):
+            raise ValidationError(
+                "Exponential distribution requires all positive values"
+            )
+        
+        # Maximum likelihood estimation: rate = 1 / mean
+        mean = np.mean(data)
+        rate = 1.0 / mean
+    
+    # Validate parameters
+    if rate <= 0:
+        raise ValidationError(f"Rate must be positive, got {rate}")
+    
+    # Generate values
+    if seed is not None:
+        np.random.seed(seed)
+    
+    # numpy uses scale = 1/rate
+    scale = 1.0 / rate
+    values = np.random.exponential(scale, n_rows)
+    
+    return values.tolist()
+
+
+def generate_kde(
+    n_rows: int,
+    data: np.ndarray,
+    bandwidth: Optional[float] = None,
+    seed: Optional[int] = None
+) -> List[float]:
+    """
+    Generate values using Kernel Density Estimation.
+    
+    KDE learns the exact distribution shape from data.
+    Non-parametric approach that preserves complex patterns.
+    
+    Args:
+        n_rows: Number of values to generate
+        data: Existing data to learn from (required)
+        bandwidth: KDE bandwidth (auto-selected if None)
+        seed: Random seed for reproducibility
+        
+    Returns:
+        List of generated values
+        
+    Raises:
+        ValidationError: If data invalid
+    """
+    if data is None or len(data) == 0:
+        raise ValidationError("KDE requires existing data")
+    
+    if len(data) < 3:
+        raise ValidationError(f"KDE requires at least 3 data points, got {len(data)}")
+    
+    # Auto-select bandwidth using Silverman's rule of thumb
+    if bandwidth is None:
+        std = np.std(data)
+        n = len(data)
+        bandwidth = 1.06 * std * (n ** (-1/5))
+        
+        # Ensure reasonable bandwidth
+        if bandwidth == 0:
+            bandwidth = 0.1 * (np.max(data) - np.min(data))
+    
+    if bandwidth <= 0:
+        raise ValidationError(f"Bandwidth must be positive, got {bandwidth}")
+    
+    # Generate values by sampling from data and adding noise
+    if seed is not None:
+        np.random.seed(seed)
+    
+    # Sample from data with replacement
+    sampled_indices = np.random.choice(len(data), size=n_rows, replace=True)
+    sampled_values = data[sampled_indices]
+    
+    # Add Gaussian noise with bandwidth as std
+    noise = np.random.normal(0, bandwidth, n_rows)
+    values = sampled_values + noise
+    
+    return values.tolist()
+
+
+def generate_multivariate_normal(
+    n_rows: int,
+    columns: List[str],
+    data: np.ndarray,
+    seed: Optional[int] = None
+) -> np.ndarray:
+    """
+    Generate correlated values using multivariate normal distribution.
+    
+    Preserves correlations between multiple columns.
+    
+    Args:
+        n_rows: Number of rows to generate
+        columns: List of column names
+        data: Existing data (2D array, shape: [n_samples, n_features])
+        seed: Random seed for reproducibility
+        
+    Returns:
+        2D array of generated values (shape: [n_rows, n_features])
+        
+    Raises:
+        ValidationError: If data invalid
+    """
+    if data is None or len(data) == 0:
+        raise ValidationError("Multivariate normal requires existing data")
+    
+    if data.ndim != 2:
+        raise ValidationError(f"Data must be 2D array, got shape {data.shape}")
+    
+    if data.shape[1] != len(columns):
+        raise ValidationError(
+            f"Number of columns ({len(columns)}) doesn't match data dimensions ({data.shape[1]})"
+        )
+    
+    # Estimate mean and covariance
+    mean = np.mean(data, axis=0)
+    cov = np.cov(data, rowvar=False)
+    
+    # Ensure covariance matrix is positive definite
+    # Add small value to diagonal if needed
+    min_eig = np.min(np.linalg.eigvals(cov))
+    if min_eig < 0:
+        cov += np.eye(cov.shape[0]) * (abs(min_eig) + 1e-6)
+    
+    # Generate values
+    if seed is not None:
+        np.random.seed(seed)
+    
+    values = np.random.multivariate_normal(mean, cov, n_rows)
+    
+    return values
+
+
+def generate_distribution_values(
+    n_rows: int,
+    distribution: str = DistributionType.AUTO,
+    data: Optional[np.ndarray] = None,
+    seed: Optional[int] = None,
+    **params
+) -> List[float]:
+    """
+    Main distribution generation function.
+    
+    Args:
+        n_rows: Number of values to generate
+        distribution: Distribution type (normal, uniform, skewed_left, skewed_right, 
+                      beta, gamma, exponential, kde, auto)
+        data: Existing data to estimate parameters from (required for auto and kde)
+        seed: Random seed for reproducibility
+        **params: Distribution-specific parameters:
+            - mean, std: For normal
+            - min_val, max_val: For uniform
+            - skewness: For skewed (default: 1.0)
+            - alpha, beta: For beta
+            - shape, scale: For gamma
+            - rate: For exponential
+            - bandwidth: For kde
+            - clip: Whether to clip to data range (default: True)
+            
+    Returns:
+        List of generated values
+        
+    Raises:
+        ValidationError: If parameters invalid
+        AugmentError: If generation fails
+    """
+    # Auto-detect distribution if requested
+    if distribution == DistributionType.AUTO or distribution == "auto":
+        if data is None:
+            raise ValidationError(
+                "Auto distribution detection requires existing data"
+            )
+        
+        distribution = detect_distribution_type(data)
+        print(f"Auto-detected distribution: {distribution}")
+    
+    # Generate based on distribution type
+    try:
+        if distribution == DistributionType.NORMAL:
+            return generate_normal(
+                n_rows,
+                mean=params.get('mean'),
+                std=params.get('std'),
+                data=data,
+                seed=seed,
+                clip=params.get('clip', True)
+            )
+        
+        elif distribution == DistributionType.UNIFORM:
+            return generate_uniform(
+                n_rows,
+                min_val=params.get('min_val'),
+                max_val=params.get('max_val'),
+                data=data,
+                seed=seed
+            )
+        
+        elif distribution in [DistributionType.SKEWED_LEFT, DistributionType.SKEWED_RIGHT]:
+            direction = 'left' if distribution == DistributionType.SKEWED_LEFT else 'right'
+            return generate_skewed(
+                n_rows,
+                direction=direction,
+                mean=params.get('mean'),
+                std=params.get('std'),
+                skewness=params.get('skewness', 1.0),
+                data=data,
+                seed=seed,
+                clip=params.get('clip', True)
+            )
+        
+        elif distribution == DistributionType.BETA:
+            return generate_beta(
+                n_rows,
+                alpha=params.get('alpha'),
+                beta_param=params.get('beta'),
+                data=data,
+                seed=seed,
+                scale_min=params.get('scale_min', 0.0),
+                scale_max=params.get('scale_max', 1.0)
+            )
+        
+        elif distribution == DistributionType.GAMMA:
+            return generate_gamma(
+                n_rows,
+                shape=params.get('shape'),
+                scale=params.get('scale'),
+                data=data,
+                seed=seed
+            )
+        
+        elif distribution == DistributionType.EXPONENTIAL:
+            return generate_exponential_dist(
+                n_rows,
+                rate=params.get('rate'),
+                data=data,
+                seed=seed
+            )
+        
+        elif distribution == DistributionType.KDE:
+            if data is None:
+                raise ValidationError("KDE requires existing data")
+            return generate_kde(
+                n_rows,
+                data=data,
+                bandwidth=params.get('bandwidth'),
+                seed=seed
+            )
+        
+        else:
+            raise ValidationError(
+                f"Unknown distribution type: '{distribution}'. "
+                f"Supported: normal, uniform, skewed_left, skewed_right, "
+                f"beta, gamma, exponential, kde, auto"
+            )
+    
+    except Exception as e:
+        if isinstance(e, (ValidationError, AugmentError)):
+            raise
+        raise AugmentError(f"Distribution generation failed: {e}")