additory 0.1.0a2__py3-none-any.whl → 0.1.0a4__py3-none-any.whl

additory/expressions/registry.py

@@ -28,9 +28,9 @@ class ResolvedFormula:
     version: str
     mode: str = "local"
     namespace: str = "builtin"  # NEW: "builtin" or "user"
-    ast: dict | None = None
-    sample_clean: dict | None = None
-    sample_unclean: dict | None = None
+    ast: Optional[dict] = None
+    sample_clean: Optional[dict] = None
+    sample_unclean: Optional[dict] = None
 
 
 # ------------------------------------------------------------

additory/synthetic/__init__.py

@@ -1,101 +1,13 @@
 """
-Additory Synthetic Data Generation Module
+Synthetic Module - Synthetic Data Generation Functionality
 
-This module provides polars-native synthetic data generation using regex patterns
-and distribution strategies. It supports hierarchical pattern resolution and
-industry-standard file formats (.properties and .toml).
+This module provides synthetic data generation capabilities to add synthetic rows
+to existing dataframes or create data from scratch by intelligently sampling 
+from existing data patterns.
 """
 
-from .api import (
-    synth, 
-    config,
-    register_distribution_engine,
-    unregister_distribution_engine,
-    list_custom_distribution_engines
-)
-from .exceptions import (
-    SyntheticDataError,
-    PatternResolutionError,
-    ValidationError,
-    DistributionError,
-    FileFormatError,
-    PatternImportError,
-    SchemaParsingError
-)
-from .pattern_resolver import PatternHierarchyResolver, ResolutionTrace, PatternResolutionResult
-from .engines import (
-    DistributionEngine,
-    DistributionEngineFactory,
-    DistributionManager,
-    DistributionConfig,
-)
-from .generator import (
-    RegexGenerator,
-    PolarsGeneratorCore,
-    OutputConverter,
-    SyntheticDataGenerator,
-    GenerationConfig,
-)
-from .performance import (
-    PerformanceMonitor,
-    PerformanceOptimizer,
-    PerformanceMetrics,
-    PerformanceComparison,
-    performance_monitor,
-    performance_optimizer
-)
-from .polars_integration import (
-    PolarsIntegrationLayer,
-    optimize_conversion,
-    enhance_result,
-    optimize_context,
-    apply_expression,
-    optimize_memory,
-    validate_compatibility,
-    get_integration_stats,
-    cleanup_integration,
-    benchmark_integration
-)
+from additory.synthetic.synthesizer import synthetic
 
 __all__ = [
-    'synth',
-    'config',
-    'register_distribution_engine',
-    'unregister_distribution_engine',
-    'list_custom_distribution_engines',
-    'SyntheticDataError',
-    'PatternResolutionError', 
-    'ValidationError',
-    'DistributionError',
-    'FileFormatError',
-    'PatternImportError',
-    'SchemaParsingError',
-    'PatternHierarchyResolver',
-    'ResolutionTrace',
-    'PatternResolutionResult',
-    'DistributionEngine',
-    'DistributionEngineFactory',
-    'DistributionManager',
-    'DistributionConfig',
-    'RegexGenerator',
-    'PolarsGeneratorCore',
-    'OutputConverter',
-    'SyntheticDataGenerator',
-    'GenerationConfig',
-    'PerformanceMonitor',
-    'PerformanceOptimizer',
-    'PerformanceMetrics',
-    'PerformanceComparison',
-    'performance_monitor',
-    'performance_optimizer',
-    'PolarsIntegrationLayer',
-    'optimize_conversion',
-    'enhance_result',
-    'optimize_context',
-    'apply_expression',
-    'optimize_memory',
-    'validate_compatibility',
-    'get_integration_stats',
-    'cleanup_integration',
-    'benchmark_integration'
-]
+    "synthetic"
+]

additory/synthetic/column_name_resolver.py

@@ -0,0 +1,149 @@
+"""
+Column Name Resolver for Linked Lists
+
+Resolves column names for linked lists using priority order:
+1. Column_Names row (explicit names)
+2. Underscore parsing from list name
+3. Fallback to {strategy_key}_1, {strategy_key}_2, etc.
+"""
+
+from typing import List, Optional
+import warnings
+
+
+def parse_column_names_from_underscores(list_name: str) -> Optional[List[str]]:
+    """
+    Parse column names from list name using underscore delimiters.
+    
+    Args:
+        list_name: Name of the list variable (e.g., "AE_CM_SEV")
+        
+    Returns:
+        List of column names, or None if no underscores found
+        
+    Examples:
+        >>> parse_column_names_from_underscores("AE_CM_SEV")
+        ['AE', 'CM', 'SEV']
+        
+        >>> parse_column_names_from_underscores("adverse_event_medication")
+        ['adverse', 'event', 'medication']
+        
+        >>> parse_column_names_from_underscores("adverseconmed")
+        None
+    """
+    if '_' not in list_name:
+        return None
+    
+    parts = list_name.split('_')
+    
+    # Filter out empty parts
+    column_names = [part for part in parts if part]
+    
+    if not column_names:
+        return None
+    
+    return column_names
+
+
+def generate_fallback_column_names(strategy_key: str, num_columns: int) -> List[str]:
+    """
+    Generate fallback column names when no other naming strategy works.
+    
+    Format: {strategy_key}_1, {strategy_key}_2, etc.
+    
+    Args:
+        strategy_key: Key from strategy dict (e.g., "col1")
+        num_columns: Number of columns to generate names for
+        
+    Returns:
+        List of column names
+        
+    Examples:
+        >>> generate_fallback_column_names("col1", 3)
+        ['col1_1', 'col1_2', 'col1_3']
+        
+        >>> generate_fallback_column_names("adverse_events", 2)
+        ['adverse_events_1', 'adverse_events_2']
+    """
+    return [f"{strategy_key}_{i+1}" for i in range(num_columns)]
+
+
+def resolve_column_names(
+    list_name: str,
+    strategy_key: str,
+    num_columns: int,
+    explicit_names: Optional[List[str]] = None
+) -> List[str]:
+    """
+    Resolve column names using priority order.
+    
+    Priority:
+    1. explicit_names (from Column_Names row)
+    2. Underscore parsing from list_name
+    3. Fallback to {strategy_key}_1, {strategy_key}_2, etc.
+    
+    Args:
+        list_name: Name of the list variable
+        strategy_key: Key from strategy dict
+        num_columns: Number of columns to generate
+        explicit_names: Explicit column names from Column_Names row (optional)
+        
+    Returns:
+        List of column names
+        
+    Raises:
+        ValueError: If explicit_names count doesn't match num_columns
+        
+    Examples:
+        >>> # Priority 1: Explicit names
+        >>> resolve_column_names("AE_CM", "col1", 2, ["adverse_event", "medication"])
+        ['adverse_event', 'medication']
+        
+        >>> # Priority 2: Underscore parsing
+        >>> resolve_column_names("AE_CM_SEV", "col1", 3)
+        ['AE', 'CM', 'SEV']
+        
+        >>> # Priority 3: Fallback
+        >>> resolve_column_names("adverseconmed", "col1", 2)
+        ['col1_1', 'col1_2']
+    """
+    # Priority 1: Explicit names from Column_Names row
+    if explicit_names is not None:
+        if len(explicit_names) != num_columns:
+            raise ValueError(
+                f"Column_Names row has {len(explicit_names)} names but "
+                f"linked list generates {num_columns} columns. They must match."
+            )
+        return explicit_names
+    
+    # Priority 2: Underscore parsing
+    parsed_names = parse_column_names_from_underscores(list_name)
+    if parsed_names is not None:
+        if len(parsed_names) == num_columns:
+            return parsed_names
+        else:
+            # Underscore count doesn't match - fall through to fallback
+            warnings.warn(
+                f"List name '{list_name}' has {len(parsed_names)} underscore-separated "
+                f"parts but generates {num_columns} columns. "
+                f"Using fallback naming: {strategy_key}_1, {strategy_key}_2, etc.\n"
+                f"Suggestion: Use a list name with {num_columns-1} underscores, "
+                f"or add a Column_Names row for explicit naming.",
+                UserWarning
+            )
+    else:
+        # No underscores - emit warning
+        warnings.warn(
+            f"List name '{list_name}' has no underscores. "
+            f"Using fallback naming: {strategy_key}_1, {strategy_key}_2, etc.\n"
+            f"Suggestion: Use underscore-delimited naming (e.g., 'AE_CM_SEV') "
+            f"or add a Column_Names row:\n"
+            f"  {list_name} = [\n"
+            f"      ['Column_Names:[col1,col2,col3]'],\n"
+            f"      ['primary', ['attr1'], ['attr2']]\n"
+            f"  ]",
+            UserWarning
+        )
+    
+    # Priority 3: Fallback
+    return generate_fallback_column_names(strategy_key, num_columns)

additory/synthetic/deduce.py

@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+"""
+Text-based label deduction for additory.
+
+Uses TF-IDF + cosine similarity to deduce labels from text.
+Pure Python, no LLMs, offline-first.
+"""
+
+import math
+import re
+from collections import Counter
+from typing import Union, List, Optional
+import pandas as pd
+import polars as pl
+
+
+def tokenize(text: str) -> List[str]:
+    """
+    Tokenize text into words.
+    
+    Args:
+        text: Input text
+        
+    Returns:
+        List of lowercase tokens
+    """
+    if text is None or not isinstance(text, str):
+        return []
+    
+    text = text.lower()
+    text = re.sub(r"[^a-z0-9\s]", " ", text)
+    return [w for w in text.split() if w]
+
+
+def vectorize(tokens: List[str]) -> Counter:
+    """
+    Convert tokens to TF vector (term frequency).
+    
+    Args:
+        tokens: List of tokens
+        
+    Returns:
+        Counter with term frequencies
+    """
+    return Counter(tokens)
+
+
+def cosine_similarity(v1: Counter, v2: Counter) -> float:
+    """
+    Compute cosine similarity between two vectors.
+    
+    Args:
+        v1: First vector (Counter)
+        v2: Second vector (Counter)
+        
+    Returns:
+        Similarity score (0-1)
+    """
+    # Dot product
+    dot = sum(v1[t] * v2[t] for t in v1 if t in v2)
+    
+    # Magnitudes
+    mag1 = math.sqrt(sum(v * v for v in v1.values()))
+    mag2 = math.sqrt(sum(v * v for v in v2.values()))
+    
+    if mag1 == 0 or mag2 == 0:
+        return 0.0
+    
+    return dot / (mag1 * mag2)
+
+
+def _deduce_polars(
+    df: pl.DataFrame,
+    from_column: Union[str, List[str]],
+    to_column: str,
+    min_examples: int = 3
+) -> pl.DataFrame:
+    """
+    Deduce missing labels using text similarity (Polars-native).
+    
+    Args:
+        df: Polars DataFrame
+        from_column: Text column(s) to analyze
+        to_column: Label column to fill
+        min_examples: Minimum labeled examples required
+        
+    Returns:
+        DataFrame with deduced labels
+        
+    Raises:
+        ValueError: If insufficient labeled examples
+    """
+    # Normalize from_column to list
+    if isinstance(from_column, str):
+        source_cols = [from_column]
+    else:
+        source_cols = from_column
+    
+    # Validate columns exist
+    for col in source_cols:
+        if col not in df.columns:
+            raise ValueError(f"Column '{col}' not found in DataFrame")
+    
+    if to_column not in df.columns:
+        raise ValueError(f"Column '{to_column}' not found in DataFrame")
+    
+    # Create combined text column if multiple sources
+    if len(source_cols) == 1:
+        text_col = source_cols[0]
+        df_work = df.clone()
+    else:
+        # Concatenate multiple columns with spaces
+        df_work = df.with_columns([
+            pl.concat_str(
+                [pl.col(c).fill_null("") for c in source_cols],
+                separator=" "
+            ).alias("__deduce_text__")
+        ])
+        text_col = "__deduce_text__"
+    
+    # Split into labeled and unlabeled
+    labeled_df = df_work.filter(pl.col(to_column).is_not_null())
+    unlabeled_df = df_work.filter(pl.col(to_column).is_null())
+    
+    # Check if we have enough labeled examples
+    n_labeled = len(labeled_df)
+    if n_labeled == 0:
+        raise ValueError(
+            f"⚠️ Cannot deduce labels: No labeled examples found in '{to_column}' column.\n"
+            f"Please manually label at least {min_examples} examples per category, then run again.\n\n"
+            f"Note: additory uses pure Python text similarity (no LLMs, no external calls).\n"
+            f"Your data never leaves your machine."
+        )
+    
+    if n_labeled < min_examples:
+        print(
+            f"⚠️ Only {n_labeled} labeled examples found. "
+            f"For better accuracy, label at least {min_examples} examples.\n"
+            f"Proceeding with available data..."
+        )
+    
+    # If no unlabeled rows, return original
+    if len(unlabeled_df) == 0:
+        if len(source_cols) > 1:
+            # Remove temporary column
+            return df_work.drop("__deduce_text__")
+        return df_work
+    
+    # Precompute vectors for labeled rows
+    labeled_vectors = []
+    for row in labeled_df.iter_rows(named=True):
+        text = row[text_col]
+        label = row[to_column]
+        tokens = tokenize(text)
+        vec = vectorize(tokens)
+        labeled_vectors.append((vec, label))
+    
+    # Deduce labels for unlabeled rows
+    deduced_labels = []
+    for row in unlabeled_df.iter_rows(named=True):
+        text = row[text_col]
+        tokens = tokenize(text)
+        vec = vectorize(tokens)
+        
+        # Find most similar labeled example
+        best_label = None
+        best_score = -1.0
+        
+        for labeled_vec, label in labeled_vectors:
+            score = cosine_similarity(vec, labeled_vec)
+            if score > best_score:
+                best_score = score
+                best_label = label
+        
+        deduced_labels.append(best_label)
+    
+    # Create deduced labels series
+    deduced_series = pl.Series(to_column, deduced_labels)
+    
+    # Update unlabeled rows with deduced labels
+    unlabeled_df = unlabeled_df.with_columns([deduced_series])
+    
+    # Combine labeled and unlabeled back together
+    result_df = pl.concat([labeled_df, unlabeled_df])
+    
+    # Remove temporary column if created
+    if len(source_cols) > 1:
+        result_df = result_df.drop("__deduce_text__")
+    
+    # Print success message
+    n_deduced = len(deduced_labels)
+    print(f"✓ Deduced {n_deduced} label{'s' if n_deduced != 1 else ''} from {n_labeled} examples (offline, no LLMs)")
+    
+    return result_df
+
+
+def deduce(
+    df: Union[pd.DataFrame, pl.DataFrame],
+    from_column: Union[str, List[str]],
+    to_column: str
+) -> Union[pd.DataFrame, pl.DataFrame]:
+    """
+    Deduce missing labels based on text similarity to labeled examples.
+    
+    Uses cosine similarity on TF-IDF vectors. Pure Python, no LLMs, offline-first.
+    Requires at least 3 labeled examples to work.
+    
+    When multiple source columns are provided, they are concatenated with
+    spaces before computing similarity.
+    
+    Args:
+        df: DataFrame with some labeled and some unlabeled rows
+        from_column: Text column(s) to analyze
+                    - str: Single column (e.g., "comment")
+                    - List[str]: Multiple columns (e.g., ["comment", "notes"])
+        to_column: Label column to fill (e.g., "status")
+        
+    Returns:
+        DataFrame with deduced labels filled in
+        
+    Examples:
+        # Single column
+        >>> result = add.deduce(df, from_column="comment", to_column="status")
+        
+        # Multiple columns (better accuracy)
+        >>> result = add.deduce(
+        ...     df, 
+        ...     from_column=["comment", "notes", "description"],
+        ...     to_column="status"
+        ... )
+    
+    Privacy: Your data never leaves your machine. No external connections.
+    """
+    # Detect input backend
+    if isinstance(df, pd.DataFrame):
+        backend = "pandas"
+        # Convert to Polars
+        df_polars = pl.from_pandas(df)
+    elif isinstance(df, pl.DataFrame):
+        backend = "polars"
+        df_polars = df
+    else:
+        # Try arrow bridge (for cudf, etc.)
+        try:
+            df_polars = pl.from_arrow(df)
+            backend = "arrow"
+        except Exception:
+            raise TypeError(f"Unsupported DataFrame type: {type(df)}")
+    
+    # Process in Polars
+    result_polars = _deduce_polars(df_polars, from_column, to_column)
+    
+    # Convert back to original format
+    if backend == "pandas":
+        return result_polars.to_pandas()
+    elif backend == "polars":
+        return result_polars
+    else:  # arrow
+        return result_polars.to_arrow()

additory/{augment → synthetic}/distributions.py

@@ -1,5 +1,5 @@
 """
-Distribution Strategies for Data Augmentation
+Distribution Strategies for Synthetic Data Generation
 
 DEPRECATED: This module has been moved to additory.common.distributions
 Please update your imports to use additory.common.distributions instead.
@@ -11,7 +11,7 @@ import warnings
 
 # Issue deprecation warning
 warnings.warn(
-    "additory.augment.distributions is deprecated. "
+    "additory.synthetic.distributions is deprecated. "
     "Please use additory.common.distributions instead. "
     "This module will be removed in a future version.",
     DeprecationWarning,

additory/{augment → synthetic}/forecast.py

@@ -1,5 +1,5 @@
 """
-Forecast Strategies for Data Augmentation
+Forecast Strategies for Synthetic Data Generation
 
 Provides time series forecasting capabilities:
 - Linear trend forecasting