additory 0.1.0a1__py3-none-any.whl

additory/__init__.py

@@ -0,0 +1,15 @@
+# additory/__init__.py
+
+from .dynamic_api import add as _api_instance
+
+# Expose the API instance normally
+add = _api_instance
+
+# Module-level __getattr__ to forward dynamic attributes
+def __getattr__(name):
+    # Delegate all unknown attributes to the API instance
+    return getattr(_api_instance, name)
+
+__all__ = [
+    "add",
+]

additory/analysis/__init__.py

@@ -0,0 +1,48 @@
+"""
+Analysis Module for Data Profiling
+
+Provides comprehensive data analysis capabilities:
+- Distribution detection and fitting
+- Correlation analysis
+- Cardinality analysis
+- Data quality metrics
+- Data profiling and scanning
+"""
+
+from additory.analysis.distributions import (
+    detect_distributions,
+    fit_distribution,
+    DistributionFit
+)
+from additory.analysis.correlations import (
+    calculate_correlations,
+    CorrelationResult
+)
+from additory.analysis.cardinality import (
+    analyze_cardinality,
+    CardinalityInfo
+)
+from additory.analysis.quality import (
+    analyze_quality,
+    QualityMetrics
+)
+from additory.analysis.scan import (
+    scan,
+    ScanResult,
+    ColumnInfo
+)
+
+__all__ = [
+    'detect_distributions',
+    'fit_distribution',
+    'DistributionFit',
+    'calculate_correlations',
+    'CorrelationResult',
+    'analyze_cardinality',
+    'CardinalityInfo',
+    'analyze_quality',
+    'QualityMetrics',
+    'scan',
+    'ScanResult',
+    'ColumnInfo',
+]

additory/analysis/cardinality.py

@@ -0,0 +1,126 @@
+"""
+Cardinality Analysis
+
+Analyzes unique values and cardinality of columns.
+"""
+
+from dataclasses import dataclass
+from typing import List, Any, Dict
+import polars as pl
+
+
+@dataclass
+class CardinalityInfo:
+    """Cardinality information for a column."""
+    unique_count: int
+    total_count: int
+    ratio: float
+    top_values: List[tuple]  # [(value, count), ...]
+    classification: str  # 'constant', 'low', 'medium', 'high'
+    
+    def __repr__(self) -> str:
+        return (
+            f"CardinalityInfo(unique={self.unique_count}, "
+            f"ratio={self.ratio:.2%}, class='{self.classification}')"
+        )
+
+
+def classify_cardinality(ratio: float, unique_count: int) -> str:
+    """
+    Classify cardinality based on ratio and unique count.
+    
+    Args:
+        ratio: Unique count / total count
+        unique_count: Number of unique values
+        
+    Returns:
+        Classification: 'constant', 'low', 'medium', 'high'
+    """
+    if unique_count == 1:
+        return 'constant'
+    elif ratio >= 0.5:
+        return 'high'
+    elif ratio >= 0.1:
+        return 'medium'
+    else:
+        return 'low'
+
+
+def analyze_cardinality(
+    df: pl.DataFrame,
+    column: str,
+    top_n: int = 10
+) -> CardinalityInfo:
+    """
+    Analyze cardinality of a column.
+    
+    Args:
+        df: Polars DataFrame
+        column: Column name
+        top_n: Number of top values to return
+        
+    Returns:
+        CardinalityInfo object
+    """
+    # Get total count (excluding nulls)
+    total_count = df[column].count()
+    
+    if total_count == 0:
+        return CardinalityInfo(
+            unique_count=0,
+            total_count=0,
+            ratio=0.0,
+            top_values=[],
+            classification='constant'
+        )
+    
+    # Get unique count (excluding nulls)
+    unique_count = df[column].drop_nulls().n_unique()
+    
+    # Calculate ratio
+    ratio = unique_count / total_count if total_count > 0 else 0.0
+    
+    # Get top values
+    value_counts = (
+        df
+        .group_by(column)
+        .agg(pl.len().alias('count'))
+        .sort('count', descending=True)
+        .head(top_n)
+    )
+    
+    top_values = [
+        (row[column], row['count'])
+        for row in value_counts.iter_rows(named=True)
+    ]
+    
+    # Classify
+    classification = classify_cardinality(ratio, unique_count)
+    
+    return CardinalityInfo(
+        unique_count=unique_count,
+        total_count=total_count,
+        ratio=ratio,
+        top_values=top_values,
+        classification=classification
+    )
+
+
+def analyze_all_cardinality(
+    df: pl.DataFrame,
+    top_n: int = 10
+) -> Dict[str, CardinalityInfo]:
+    """
+    Analyze cardinality for all columns.
+    
+    Args:
+        df: Polars DataFrame
+        top_n: Number of top values to return per column
+        
+    Returns:
+        Dictionary mapping column names to CardinalityInfo
+    """
+    return {
+        col: analyze_cardinality(df, col, top_n)
+        for col in df.columns
+    }

additory/analysis/correlations.py

@@ -0,0 +1,124 @@
+"""
+Correlation Analysis
+
+Calculates correlations between numeric columns.
+"""
+
+from dataclasses import dataclass
+from typing import Dict, List, Tuple
+import numpy as np
+import polars as pl
+from scipy import stats
+
+
+@dataclass
+class CorrelationResult:
+    """Result of correlation analysis between two columns."""
+    column1: str
+    column2: str
+    correlation: float
+    method: str
+    p_value: float = 0.0
+
+
+def calculate_correlations(
+    df: pl.DataFrame,
+    columns: List[str],
+    methods: List[str] = ['pearson', 'spearman'],
+    threshold: float = 0.0
+) -> List[CorrelationResult]:
+    """
+    Calculate correlations between numeric columns with optimized batch processing.
+    
+    Args:
+        df: Polars DataFrame
+        columns: List of numeric column names
+        methods: Correlation methods to calculate
+        threshold: Minimum correlation threshold to report
+        
+    Returns:
+        List of CorrelationResult objects (changed from single object for scan.py compatibility)
+    """
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    import itertools
+    
+    if len(columns) < 2:
+        return []
+    
+    # Pre-extract all data as numpy arrays for efficiency
+    data_arrays = {}
+    for col in columns:
+        arr = df[col].to_numpy()
+        data_arrays[col] = arr
+    
+    # Generate all column pairs
+    column_pairs = list(itertools.combinations(columns, 2))
+    
+    results = []
+    
+    def calculate_pair_correlations(pair):
+        """Calculate correlations for a single pair of columns."""
+        col1, col2 = pair
+        arr1 = data_arrays[col1]
+        arr2 = data_arrays[col2]
+        
+        # Get common non-NaN indices
+        mask = ~(np.isnan(arr1) | np.isnan(arr2))
+        arr1_clean = arr1[mask]
+        arr2_clean = arr2[mask]
+        
+        if len(arr1_clean) < 3:
+            return None
+        
+        pair_results = {}
+        
+        # Calculate all requested methods for this pair
+        for method in methods:
+            try:
+                if method == 'pearson':
+                    corr, p_value = stats.pearsonr(arr1_clean, arr2_clean)
+                elif method == 'spearman':
+                    corr, p_value = stats.spearmanr(arr1_clean, arr2_clean)
+                elif method == 'kendall':
+                    corr, p_value = stats.kendalltau(arr1_clean, arr2_clean)
+                else:
+                    continue
+                
+                # Only include if above threshold
+                if abs(corr) >= threshold:
+                    pair_results[method] = {
+                        'correlation': float(corr),
+                        'p_value': float(p_value)
+                    }
+            except Exception:
+                continue
+        
+        if pair_results:
+            return (col1, col2, pair_results)
+        return None
+    
+    # Use ThreadPoolExecutor for parallel processing of correlation pairs
+    with ThreadPoolExecutor(max_workers=min(4, len(column_pairs))) as executor:
+        # Submit all pair processing tasks
+        future_to_pair = {
+            executor.submit(calculate_pair_correlations, pair): pair 
+            for pair in column_pairs
+        }
+        
+        # Collect results as they complete
+        for future in as_completed(future_to_pair):
+            result = future.result()
+            if result is not None:
+                col1, col2, pair_results = result
+                
+                # Create CorrelationResult objects for each method
+                for method, corr_data in pair_results.items():
+                    results.append(CorrelationResult(
+                        column1=col1,
+                        column2=col2,
+                        correlation=corr_data['correlation'],
+                        method=method,
+                        p_value=corr_data['p_value']
+                    ))
+    
+    return results

additory/analysis/distributions.py

@@ -0,0 +1,376 @@
+"""
+Distribution Detection and Fitting
+
+Detects and fits statistical distributions to numeric data.
+"""
+
+from dataclasses import dataclass
+from typing import List, Dict, Any, Optional
+import numpy as np
+from scipy import stats
+
+
+@dataclass
+class DistributionFit:
+    """Result of fitting a distribution to data."""
+    name: str
+    params: Dict[str, float]
+    goodness_of_fit: float  # KS test statistic (lower is better)
+    p_value: float  # KS test p-value (higher is better)
+    
+    def __repr__(self) -> str:
+        return f"DistributionFit(name='{self.name}', fit={self.goodness_of_fit:.4f}, p={self.p_value:.4f})"
+
+
+def fit_normal(data: np.ndarray) -> DistributionFit:
+    """Fit normal distribution."""
+    mean, std = stats.norm.fit(data)
+    ks_stat, p_value = stats.kstest(data, 'norm', args=(mean, std))
+    
+    return DistributionFit(
+        name='normal',
+        params={'mean': float(mean), 'std': float(std)},
+        goodness_of_fit=float(ks_stat),
+        p_value=float(p_value)
+    )
+
+
+def fit_uniform(data: np.ndarray) -> DistributionFit:
+    """Fit uniform distribution."""
+    loc, scale = stats.uniform.fit(data)
+    ks_stat, p_value = stats.kstest(data, 'uniform', args=(loc, scale))
+    
+    return DistributionFit(
+        name='uniform',
+        params={'min': float(loc), 'max': float(loc + scale)},
+        goodness_of_fit=float(ks_stat),
+        p_value=float(p_value)
+    )
+
+
+def fit_exponential(data: np.ndarray) -> Optional[DistributionFit]:
+    """Fit exponential distribution (requires positive values)."""
+    if np.any(data <= 0):
+        return None
+    
+    loc, scale = stats.expon.fit(data)
+    ks_stat, p_value = stats.kstest(data, 'expon', args=(loc, scale))
+    
+    return DistributionFit(
+        name='exponential',
+        params={'loc': float(loc), 'scale': float(scale), 'rate': float(1/scale)},
+        goodness_of_fit=float(ks_stat),
+        p_value=float(p_value)
+    )
+
+
+def fit_lognormal(data: np.ndarray) -> Optional[DistributionFit]:
+    """Fit log-normal distribution (requires positive values)."""
+    if np.any(data <= 0):
+        return None
+    
+    shape, loc, scale = stats.lognorm.fit(data, floc=0)
+    ks_stat, p_value = stats.kstest(data, 'lognorm', args=(shape, loc, scale))
+    
+    return DistributionFit(
+        name='lognormal',
+        params={'shape': float(shape), 'loc': float(loc), 'scale': float(scale)},
+        goodness_of_fit=float(ks_stat),
+        p_value=float(p_value)
+    )
+
+
+def fit_gamma(data: np.ndarray) -> Optional[DistributionFit]:
+    """Fit gamma distribution (requires positive values)."""
+    if np.any(data <= 0):
+        return None
+    
+    shape, loc, scale = stats.gamma.fit(data, floc=0)
+    ks_stat, p_value = stats.kstest(data, 'gamma', args=(shape, loc, scale))
+    
+    return DistributionFit(
+        name='gamma',
+        params={'shape': float(shape), 'loc': float(loc), 'scale': float(scale)},
+        goodness_of_fit=float(ks_stat),
+        p_value=float(p_value)
+    )
+
+
+def fit_beta(data: np.ndarray) -> Optional[DistributionFit]:
+    """Fit beta distribution (requires values in [0, 1] or will be normalized)."""
+    # Normalize to [0, 1]
+    data_min, data_max = np.min(data), np.max(data)
+    
+    if data_max == data_min:
+        return None
+    
+    normalized = (data - data_min) / (data_max - data_min)
+    
+    # Avoid exact 0 and 1 for beta fitting
+    normalized = np.clip(normalized, 1e-6, 1 - 1e-6)
+    
+    a, b, loc, scale = stats.beta.fit(normalized, floc=0, fscale=1)
+    ks_stat, p_value = stats.kstest(normalized, 'beta', args=(a, b, loc, scale))
+    
+    return DistributionFit(
+        name='beta',
+        params={
+            'alpha': float(a),
+            'beta': float(b),
+            'data_min': float(data_min),
+            'data_max': float(data_max)
+        },
+        goodness_of_fit=float(ks_stat),
+        p_value=float(p_value)
+    )
+
+
+def fit_poisson(data: np.ndarray) -> Optional[DistributionFit]:
+    """Fit Poisson distribution (requires non-negative integers)."""
+    # Check if data looks like integers
+    if not np.allclose(data, np.round(data)):
+        return None
+    
+    if np.any(data < 0):
+        return None
+    
+    mu = np.mean(data)
+    
+    # For Poisson, use chi-square test instead of KS
+    # KS test doesn't work well for discrete distributions
+    # We'll use a simplified goodness-of-fit measure
+    expected_var = mu
+    actual_var = np.var(data)
+    
+    # Goodness of fit: how close variance is to mean (Poisson property)
+    if mu > 0:
+        fit_score = abs(actual_var - expected_var) / mu
+    else:
+        fit_score = 1.0
+    
+    return DistributionFit(
+        name='poisson',
+        params={'lambda': float(mu)},
+        goodness_of_fit=float(fit_score),
+        p_value=0.0  # Not applicable for this simplified test
+    )
+
+
+def fit_chisquare(data: np.ndarray) -> Optional[DistributionFit]:
+    """Fit chi-squared distribution (requires positive values)."""
+    if np.any(data <= 0):
+        return None
+    
+    df, loc, scale = stats.chi2.fit(data, floc=0)
+    ks_stat, p_value = stats.kstest(data, 'chi2', args=(df, loc, scale))
+    
+    return DistributionFit(
+        name='chisquare',
+        params={'df': float(df), 'loc': float(loc), 'scale': float(scale)},
+        goodness_of_fit=float(ks_stat),
+        p_value=float(p_value)
+    )
+
+
+def fit_distribution(data: np.ndarray, dist_name: str) -> Optional[DistributionFit]:
+    """
+    Fit a specific distribution to data.
+    
+    Args:
+        data: Numeric data array
+        dist_name: Distribution name (normal, uniform, exponential, etc.)
+        
+    Returns:
+        DistributionFit object or None if fitting failed
+    """
+    if len(data) < 3:
+        return None
+    
+    # Remove NaN values
+    data = data[~np.isnan(data)]
+    
+    if len(data) < 3:
+        return None
+    
+    try:
+        if dist_name == 'normal':
+            return fit_normal(data)
+        elif dist_name == 'uniform':
+            return fit_uniform(data)
+        elif dist_name == 'exponential':
+            return fit_exponential(data)
+        elif dist_name == 'lognormal':
+            return fit_lognormal(data)
+        elif dist_name == 'gamma':
+            return fit_gamma(data)
+        elif dist_name == 'beta':
+            return fit_beta(data)
+        elif dist_name == 'poisson':
+            return fit_poisson(data)
+        elif dist_name == 'chisquare':
+            return fit_chisquare(data)
+        else:
+            return None
+    except Exception:
+        return None
+
+
+def detect_distributions(
+    data: np.ndarray,
+    top_n: int = 3
+) -> List[DistributionFit]:
+    """
+    Detect best-fitting distributions for data.
+    
+    Args:
+        data: Numeric data array
+        top_n: Number of top distributions to return
+        
+    Returns:
+        List of DistributionFit objects, sorted by goodness of fit
+    """
+    if len(data) < 3:
+        return []
+    
+    # Remove NaN values
+    data = data[~np.isnan(data)]
+    
+    if len(data) < 3:
+        return []
+    
+    # Try all distributions
+    distributions = [
+        'normal',
+        'uniform',
+        'exponential',
+        'lognormal',
+        'gamma',
+        'beta',
+        'poisson',
+        'chisquare'
+    ]
+    
+    fits = []
+    for dist_name in distributions:
+        fit = fit_distribution(data, dist_name)
+        if fit is not None:
+            fits.append(fit)
+    
+    # Sort by goodness of fit (lower is better)
+    fits.sort(key=lambda x: x.goodness_of_fit)
+    
+    return fits[:top_n]
+
+
+def detect_distributions(
+    df,
+    columns: List[str] = None,
+    top_n: int = 3
+) -> Dict[str, List[DistributionFit]]:
+    """
+    Detect best-fitting distributions for multiple columns in a DataFrame.
+    
+    Args:
+        df: Polars DataFrame
+        columns: List of column names to analyze (None = all numeric columns)
+        top_n: Number of top distributions to return per column
+        
+    Returns:
+        Dictionary mapping column names to lists of DistributionFit objects
+    """
+    import polars as pl
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    import numpy as np
+    
+    if columns is None:
+        # Auto-detect numeric columns
+        columns = [col for col in df.columns 
+                  if df[col].dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                                      pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                                      pl.Float32, pl.Float64]]
+    
+    results = {}
+    
+    def process_column(col_name):
+        """Process a single column for distribution detection"""
+        try:
+            # Extract column data as numpy array
+            col_data = df[col_name].to_numpy()
+            
+            # Remove null values
+            col_data = col_data[~np.isnan(col_data)]
+            
+            if len(col_data) < 3:
+                return col_name, []
+            
+            # Detect distributions for this column
+            fits = detect_distributions_array(col_data, top_n)
+            return col_name, fits
+            
+        except Exception as e:
+            # Log error but continue with other columns
+            return col_name, []
+    
+    # Use ThreadPoolExecutor for parallel processing
+    with ThreadPoolExecutor(max_workers=min(4, len(columns))) as executor:
+        # Submit all column processing tasks
+        future_to_column = {
+            executor.submit(process_column, col): col 
+            for col in columns
+        }
+        
+        # Collect results as they complete
+        for future in as_completed(future_to_column):
+            col_name, fits = future.result()
+            results[col_name] = fits
+    
+    return results
+
+
+def detect_distributions_array(
+    data: np.ndarray,
+    top_n: int = 3
+) -> List[DistributionFit]:
+    """
+    Detect best-fitting distributions for data array.
+    
+    This is the original function renamed to avoid conflicts.
+    
+    Args:
+        data: Numeric data array
+        top_n: Number of top distributions to return
+        
+    Returns:
+        List of DistributionFit objects, sorted by goodness of fit
+    """
+    if len(data) < 3:
+        return []
+    
+    # Remove NaN values
+    data = data[~np.isnan(data)]
+    
+    if len(data) < 3:
+        return []
+    
+    # Try all distributions
+    distributions = [
+        'normal',
+        'uniform',
+        'exponential',
+        'lognormal',
+        'gamma',
+        'beta',
+        'poisson',
+        'chisquare'
+    ]
+    
+    fits = []
+    for dist_name in distributions:
+        fit = fit_distribution(data, dist_name)
+        if fit is not None:
+            fits.append(fit)
+    
+    # Sort by goodness of fit (lower is better)
+    fits.sort(key=lambda x: x.goodness_of_fit)
+    
+    return fits[:top_n]