additory 0.1.0a3__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/analysis/distributions.py

@@ -1,376 +0,0 @@
-"""
-Distribution Detection and Fitting
-
-Detects and fits statistical distributions to numeric data.
-"""
-
-from dataclasses import dataclass
-from typing import List, Dict, Any, Optional
-import numpy as np
-from scipy import stats
-
-
-@dataclass
-class DistributionFit:
-    """Result of fitting a distribution to data."""
-    name: str
-    params: Dict[str, float]
-    goodness_of_fit: float  # KS test statistic (lower is better)
-    p_value: float  # KS test p-value (higher is better)
-    
-    def __repr__(self) -> str:
-        return f"DistributionFit(name='{self.name}', fit={self.goodness_of_fit:.4f}, p={self.p_value:.4f})"
-
-
-def fit_normal(data: np.ndarray) -> DistributionFit:
-    """Fit normal distribution."""
-    mean, std = stats.norm.fit(data)
-    ks_stat, p_value = stats.kstest(data, 'norm', args=(mean, std))
-    
-    return DistributionFit(
-        name='normal',
-        params={'mean': float(mean), 'std': float(std)},
-        goodness_of_fit=float(ks_stat),
-        p_value=float(p_value)
-    )
-
-
-def fit_uniform(data: np.ndarray) -> DistributionFit:
-    """Fit uniform distribution."""
-    loc, scale = stats.uniform.fit(data)
-    ks_stat, p_value = stats.kstest(data, 'uniform', args=(loc, scale))
-    
-    return DistributionFit(
-        name='uniform',
-        params={'min': float(loc), 'max': float(loc + scale)},
-        goodness_of_fit=float(ks_stat),
-        p_value=float(p_value)
-    )
-
-
-def fit_exponential(data: np.ndarray) -> Optional[DistributionFit]:
-    """Fit exponential distribution (requires positive values)."""
-    if np.any(data <= 0):
-        return None
-    
-    loc, scale = stats.expon.fit(data)
-    ks_stat, p_value = stats.kstest(data, 'expon', args=(loc, scale))
-    
-    return DistributionFit(
-        name='exponential',
-        params={'loc': float(loc), 'scale': float(scale), 'rate': float(1/scale)},
-        goodness_of_fit=float(ks_stat),
-        p_value=float(p_value)
-    )
-
-
-def fit_lognormal(data: np.ndarray) -> Optional[DistributionFit]:
-    """Fit log-normal distribution (requires positive values)."""
-    if np.any(data <= 0):
-        return None
-    
-    shape, loc, scale = stats.lognorm.fit(data, floc=0)
-    ks_stat, p_value = stats.kstest(data, 'lognorm', args=(shape, loc, scale))
-    
-    return DistributionFit(
-        name='lognormal',
-        params={'shape': float(shape), 'loc': float(loc), 'scale': float(scale)},
-        goodness_of_fit=float(ks_stat),
-        p_value=float(p_value)
-    )
-
-
-def fit_gamma(data: np.ndarray) -> Optional[DistributionFit]:
-    """Fit gamma distribution (requires positive values)."""
-    if np.any(data <= 0):
-        return None
-    
-    shape, loc, scale = stats.gamma.fit(data, floc=0)
-    ks_stat, p_value = stats.kstest(data, 'gamma', args=(shape, loc, scale))
-    
-    return DistributionFit(
-        name='gamma',
-        params={'shape': float(shape), 'loc': float(loc), 'scale': float(scale)},
-        goodness_of_fit=float(ks_stat),
-        p_value=float(p_value)
-    )
-
-
-def fit_beta(data: np.ndarray) -> Optional[DistributionFit]:
-    """Fit beta distribution (requires values in [0, 1] or will be normalized)."""
-    # Normalize to [0, 1]
-    data_min, data_max = np.min(data), np.max(data)
-    
-    if data_max == data_min:
-        return None
-    
-    normalized = (data - data_min) / (data_max - data_min)
-    
-    # Avoid exact 0 and 1 for beta fitting
-    normalized = np.clip(normalized, 1e-6, 1 - 1e-6)
-    
-    a, b, loc, scale = stats.beta.fit(normalized, floc=0, fscale=1)
-    ks_stat, p_value = stats.kstest(normalized, 'beta', args=(a, b, loc, scale))
-    
-    return DistributionFit(
-        name='beta',
-        params={
-            'alpha': float(a),
-            'beta': float(b),
-            'data_min': float(data_min),
-            'data_max': float(data_max)
-        },
-        goodness_of_fit=float(ks_stat),
-        p_value=float(p_value)
-    )
-
-
-def fit_poisson(data: np.ndarray) -> Optional[DistributionFit]:
-    """Fit Poisson distribution (requires non-negative integers)."""
-    # Check if data looks like integers
-    if not np.allclose(data, np.round(data)):
-        return None
-    
-    if np.any(data < 0):
-        return None
-    
-    mu = np.mean(data)
-    
-    # For Poisson, use chi-square test instead of KS
-    # KS test doesn't work well for discrete distributions
-    # We'll use a simplified goodness-of-fit measure
-    expected_var = mu
-    actual_var = np.var(data)
-    
-    # Goodness of fit: how close variance is to mean (Poisson property)
-    if mu > 0:
-        fit_score = abs(actual_var - expected_var) / mu
-    else:
-        fit_score = 1.0
-    
-    return DistributionFit(
-        name='poisson',
-        params={'lambda': float(mu)},
-        goodness_of_fit=float(fit_score),
-        p_value=0.0  # Not applicable for this simplified test
-    )
-
-
-def fit_chisquare(data: np.ndarray) -> Optional[DistributionFit]:
-    """Fit chi-squared distribution (requires positive values)."""
-    if np.any(data <= 0):
-        return None
-    
-    df, loc, scale = stats.chi2.fit(data, floc=0)
-    ks_stat, p_value = stats.kstest(data, 'chi2', args=(df, loc, scale))
-    
-    return DistributionFit(
-        name='chisquare',
-        params={'df': float(df), 'loc': float(loc), 'scale': float(scale)},
-        goodness_of_fit=float(ks_stat),
-        p_value=float(p_value)
-    )
-
-
-def fit_distribution(data: np.ndarray, dist_name: str) -> Optional[DistributionFit]:
-    """
-    Fit a specific distribution to data.
-    
-    Args:
-        data: Numeric data array
-        dist_name: Distribution name (normal, uniform, exponential, etc.)
-        
-    Returns:
-        DistributionFit object or None if fitting failed
-    """
-    if len(data) < 3:
-        return None
-    
-    # Remove NaN values
-    data = data[~np.isnan(data)]
-    
-    if len(data) < 3:
-        return None
-    
-    try:
-        if dist_name == 'normal':
-            return fit_normal(data)
-        elif dist_name == 'uniform':
-            return fit_uniform(data)
-        elif dist_name == 'exponential':
-            return fit_exponential(data)
-        elif dist_name == 'lognormal':
-            return fit_lognormal(data)
-        elif dist_name == 'gamma':
-            return fit_gamma(data)
-        elif dist_name == 'beta':
-            return fit_beta(data)
-        elif dist_name == 'poisson':
-            return fit_poisson(data)
-        elif dist_name == 'chisquare':
-            return fit_chisquare(data)
-        else:
-            return None
-    except Exception:
-        return None
-
-
-def detect_distributions(
-    data: np.ndarray,
-    top_n: int = 3
-) -> List[DistributionFit]:
-    """
-    Detect best-fitting distributions for data.
-    
-    Args:
-        data: Numeric data array
-        top_n: Number of top distributions to return
-        
-    Returns:
-        List of DistributionFit objects, sorted by goodness of fit
-    """
-    if len(data) < 3:
-        return []
-    
-    # Remove NaN values
-    data = data[~np.isnan(data)]
-    
-    if len(data) < 3:
-        return []
-    
-    # Try all distributions
-    distributions = [
-        'normal',
-        'uniform',
-        'exponential',
-        'lognormal',
-        'gamma',
-        'beta',
-        'poisson',
-        'chisquare'
-    ]
-    
-    fits = []
-    for dist_name in distributions:
-        fit = fit_distribution(data, dist_name)
-        if fit is not None:
-            fits.append(fit)
-    
-    # Sort by goodness of fit (lower is better)
-    fits.sort(key=lambda x: x.goodness_of_fit)
-    
-    return fits[:top_n]
-
-
-def detect_distributions(
-    df,
-    columns: List[str] = None,
-    top_n: int = 3
-) -> Dict[str, List[DistributionFit]]:
-    """
-    Detect best-fitting distributions for multiple columns in a DataFrame.
-    
-    Args:
-        df: Polars DataFrame
-        columns: List of column names to analyze (None = all numeric columns)
-        top_n: Number of top distributions to return per column
-        
-    Returns:
-        Dictionary mapping column names to lists of DistributionFit objects
-    """
-    import polars as pl
-    from concurrent.futures import ThreadPoolExecutor, as_completed
-    import numpy as np
-    
-    if columns is None:
-        # Auto-detect numeric columns
-        columns = [col for col in df.columns 
-                  if df[col].dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64,
-                                      pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
-                                      pl.Float32, pl.Float64]]
-    
-    results = {}
-    
-    def process_column(col_name):
-        """Process a single column for distribution detection"""
-        try:
-            # Extract column data as numpy array
-            col_data = df[col_name].to_numpy()
-            
-            # Remove null values
-            col_data = col_data[~np.isnan(col_data)]
-            
-            if len(col_data) < 3:
-                return col_name, []
-            
-            # Detect distributions for this column
-            fits = detect_distributions_array(col_data, top_n)
-            return col_name, fits
-            
-        except Exception as e:
-            # Log error but continue with other columns
-            return col_name, []
-    
-    # Use ThreadPoolExecutor for parallel processing
-    with ThreadPoolExecutor(max_workers=min(4, len(columns))) as executor:
-        # Submit all column processing tasks
-        future_to_column = {
-            executor.submit(process_column, col): col 
-            for col in columns
-        }
-        
-        # Collect results as they complete
-        for future in as_completed(future_to_column):
-            col_name, fits = future.result()
-            results[col_name] = fits
-    
-    return results
-
-
-def detect_distributions_array(
-    data: np.ndarray,
-    top_n: int = 3
-) -> List[DistributionFit]:
-    """
-    Detect best-fitting distributions for data array.
-    
-    This is the original function renamed to avoid conflicts.
-    
-    Args:
-        data: Numeric data array
-        top_n: Number of top distributions to return
-        
-    Returns:
-        List of DistributionFit objects, sorted by goodness of fit
-    """
-    if len(data) < 3:
-        return []
-    
-    # Remove NaN values
-    data = data[~np.isnan(data)]
-    
-    if len(data) < 3:
-        return []
-    
-    # Try all distributions
-    distributions = [
-        'normal',
-        'uniform',
-        'exponential',
-        'lognormal',
-        'gamma',
-        'beta',
-        'poisson',
-        'chisquare'
-    ]
-    
-    fits = []
-    for dist_name in distributions:
-        fit = fit_distribution(data, dist_name)
-        if fit is not None:
-            fits.append(fit)
-    
-    # Sort by goodness of fit (lower is better)
-    fits.sort(key=lambda x: x.goodness_of_fit)
-    
-    return fits[:top_n]

additory/analysis/quality.py

@@ -1,158 +0,0 @@
-"""
-Data Quality Metrics
-
-Analyzes data quality including missing values, types, and statistics.
-"""
-
-from dataclasses import dataclass
-from typing import Optional, Any, Dict
-import polars as pl
-import numpy as np
-
-
-@dataclass
-class QualityMetrics:
-    """Data quality metrics for a column."""
-    column: str
-    dtype: str
-    missing_count: int
-    missing_ratio: float
-    total_count: int
-    
-    # Numeric statistics
-    min_value: Optional[float] = None
-    max_value: Optional[float] = None
-    mean: Optional[float] = None
-    median: Optional[float] = None
-    std: Optional[float] = None
-    q25: Optional[float] = None
-    q75: Optional[float] = None
-    
-    # Categorical statistics
-    mode: Optional[Any] = None
-    mode_count: Optional[int] = None
-    mode_ratio: Optional[float] = None
-    
-    def __repr__(self) -> str:
-        return (
-            f"QualityMetrics(column='{self.column}', "
-            f"dtype='{self.dtype}', missing={self.missing_ratio:.1%})"
-        )
-    
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary."""
-        return {
-            'column': self.column,
-            'dtype': self.dtype,
-            'missing_count': self.missing_count,
-            'missing_ratio': self.missing_ratio,
-            'total_count': self.total_count,
-            'min': self.min_value,
-            'max': self.max_value,
-            'mean': self.mean,
-            'median': self.median,
-            'std': self.std,
-            'q25': self.q25,
-            'q75': self.q75,
-            'mode': self.mode,
-            'mode_count': self.mode_count,
-            'mode_ratio': self.mode_ratio
-        }
-
-
-def is_numeric_dtype(dtype: pl.DataType) -> bool:
-    """Check if dtype is numeric."""
-    return dtype in [
-        pl.Int8, pl.Int16, pl.Int32, pl.Int64,
-        pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
-        pl.Float32, pl.Float64
-    ]
-
-
-def analyze_quality(
-    df: pl.DataFrame,
-    column: str
-) -> QualityMetrics:
-    """
-    Analyze data quality for a column.
-    
-    Args:
-        df: Polars DataFrame
-        column: Column name
-        
-    Returns:
-        QualityMetrics object
-    """
-    col_series = df[column]
-    dtype = col_series.dtype
-    
-    # Basic counts
-    total_count = len(df)
-    missing_count = col_series.null_count()
-    missing_ratio = missing_count / total_count if total_count > 0 else 0.0
-    
-    # Initialize metrics
-    metrics = QualityMetrics(
-        column=column,
-        dtype=str(dtype),
-        missing_count=missing_count,
-        missing_ratio=missing_ratio,
-        total_count=total_count
-    )
-    
-    # Numeric statistics
-    if is_numeric_dtype(dtype):
-        try:
-            metrics.min_value = float(col_series.min())
-            metrics.max_value = float(col_series.max())
-            metrics.mean = float(col_series.mean())
-            metrics.median = float(col_series.median())
-            metrics.std = float(col_series.std())
-            
-            # Quantiles
-            q25 = col_series.quantile(0.25, interpolation='linear')
-            q75 = col_series.quantile(0.75, interpolation='linear')
-            if q25 is not None:
-                metrics.q25 = float(q25)
-            if q75 is not None:
-                metrics.q75 = float(q75)
-        except Exception:
-            pass
-    
-    # Mode (for all types)
-    try:
-        mode_result = (
-            df
-            .group_by(column)
-            .agg(pl.len().alias('count'))
-            .sort('count', descending=True)
-            .head(1)
-        )
-        
-        if len(mode_result) > 0:
-            row = mode_result.row(0, named=True)
-            metrics.mode = row[column]
-            metrics.mode_count = row['count']
-            metrics.mode_ratio = metrics.mode_count / total_count if total_count > 0 else 0.0
-    except Exception:
-        pass
-    
-    return metrics
-
-
-def analyze_all_quality(
-    df: pl.DataFrame
-) -> Dict[str, QualityMetrics]:
-    """
-    Analyze data quality for all columns.
-    
-    Args:
-        df: Polars DataFrame
-        
-    Returns:
-        Dictionary mapping column names to QualityMetrics
-    """
-    return {
-        col: analyze_quality(df, col)
-        for col in df.columns
-    }