additory 0.1.0a2__py3-none-any.whl → 0.1.0a4__py3-none-any.whl

additory/{augment → synthetic}/strategies.py

@@ -1,11 +1,10 @@
 """
-Strategy handlers for data augmentation
+Strategy handlers for synthetic data generation
 
 Provides different strategies for generating synthetic data:
 - auto: Random sampling from existing values
 - increment: Increment numeric or pattern-based values
 - choice:[...]: Random selection from inline list
-- choice_list:name: Random selection from registered/built-in list
 """
 
 import re
@@ -13,7 +12,6 @@ import random
 from typing import Any, Dict, List, Optional, Tuple
 
 from additory.common.exceptions import ValidationError, AugmentError
-from additory.augment.list_registry import get_list
 
 
 def parse_strategy_params(strategy_spec: str) -> Tuple[str, Dict[str, Any]]:
@@ -319,7 +317,7 @@ def apply_increment_strategy(
     Apply increment strategy to a column (Polars-only).
     
     Supports two modes:
-    1. Augment mode: Increment from last value in df_polars
+    1. Extend mode: Increment from last value in df_polars
     2. Create mode: Start from specified value (requires params with 'start')
     
     Args:
@@ -336,7 +334,7 @@ def apply_increment_strategy(
         ValidationError: If strategy cannot be applied
     
     Examples:
-        # Augment mode (with DataFrame)
+        # Extend mode (with DataFrame)
         >>> apply_increment_strategy(df, "id", "increment", 5)
         [11, 12, 13, 14, 15]  # if last value was 10
         
@@ -349,7 +347,7 @@ def apply_increment_strategy(
         ...                          {"start": 1, "pattern": "EMP_[001]"})
         ["EMP_001", "EMP_002", "EMP_003"]
     """
-    # Determine mode: augment (has df) or create (no df)
+    # Determine mode: extend (has df) or create (no df)
     is_create_mode = df_polars is None
     
     if is_create_mode:
@@ -402,7 +400,7 @@ def apply_increment_strategy(
         return new_values
     
     else:
-        # Augment mode: use existing logic
+        # Extend mode: use existing logic
         # Parse the strategy
         pattern, regex_pattern = parse_increment_strategy(strategy_spec)
         
@@ -483,12 +481,11 @@ def parse_choice_strategy(strategy_spec: str) -> Tuple[str, Optional[List[Any]]]
     Args:
         strategy_spec: Strategy string like:
             - "choice:[value1,value2,value3]"
-            - "choice_list:banks"
     
     Returns:
         Tuple of (strategy_type, values)
-        - strategy_type: "choice" or "choice_list"
-        - values: List of values (for choice) or None (for choice_list)
+        - strategy_type: "choice"
+        - values: List of values
     
     Raises:
         ValidationError: If strategy format is invalid
@@ -496,9 +493,6 @@ def parse_choice_strategy(strategy_spec: str) -> Tuple[str, Optional[List[Any]]]
     Examples:
         >>> parse_choice_strategy("choice:[Active,Inactive,Pending]")
         ("choice", ["Active", "Inactive", "Pending"])
-        
-        >>> parse_choice_strategy("choice_list:banks")
-        ("choice_list", None)
     """
     if strategy_spec.startswith("choice:["):
         # Inline list: choice:[value1,value2,value3]
@@ -526,22 +520,10 @@ def parse_choice_strategy(strategy_spec: str) -> Tuple[str, Optional[List[Any]]]
         
         return "choice", values
     
-    elif strategy_spec.startswith("choice_list:"):
-        # Named list: choice_list:banks
-        list_name = strategy_spec[len("choice_list:"):].strip()
-        
-        if not list_name:
-            raise ValidationError(
-                f"Invalid choice_list strategy: {strategy_spec}. "
-                "Must be in format: choice_list:list_name"
-            )
-        
-        return "choice_list", list_name
-    
     else:
         raise ValidationError(
             f"Invalid choice strategy: {strategy_spec}. "
-            "Must start with 'choice:[' or 'choice_list:'"
+            "Must start with 'choice:['"
         )
 
 
@@ -609,22 +591,7 @@ def apply_choice_strategy(
         ValidationError: If strategy cannot be applied
     """
     # Parse the strategy
-    strategy_type, values_or_name = parse_choice_strategy(strategy_spec)
-    
-    # Get the actual values list
-    if strategy_type == "choice":
-        values = values_or_name
-    elif strategy_type == "choice_list":
-        # Resolve list name to actual list
-        list_name = values_or_name
-        try:
-            values = get_list(list_name)
-        except ValidationError as e:
-            raise ValidationError(
-                f"Cannot apply choice_list strategy: {e}"
-            )
-    else:
-        raise ValidationError(f"Unknown choice strategy type: {strategy_type}")
+    strategy_type, values = parse_choice_strategy(strategy_spec)
     
     # Generate random selections
     if seed is not None:
@@ -675,7 +642,7 @@ def apply_forecast_strategy(
         >>> apply_forecast_strategy(df, "sales", "forecast:seasonal:period=12", 24)
         [98.5, 102.3, 95.8, ...]
     """
-    from additory.augment.forecast import forecast_values, ForecastMethod
+    from additory.synthetic.forecast import forecast_values, ForecastMethod
     
     # Parse strategy: forecast:method:param1=val1:param2=val2
     parts = strategy_spec.split(":")
@@ -845,7 +812,7 @@ def apply_smote_strategy(
         >>> apply_smote_strategy(df, ["feature1", "feature2"], "smote:k=5", 100)
         {"feature1": [1.2, 3.4, ...], "feature2": [5.6, 7.8, ...]}
     """
-    from additory.augment.smote import generate_smote_values
+    from additory.synthetic.smote import generate_smote_values
     
     # Parse strategy: smote:k=5
     parts = strategy_spec.split(":")
@@ -881,3 +848,79 @@ def apply_smote_strategy(
         )
     except Exception as e:
         raise ValidationError(f"SMOTE strategy failed: {e}")
+
+
+def parse_deduce_strategy(strategy_spec: str) -> Tuple[str, List[str]]:
+    """
+    Parse deduce strategy specification.
+    
+    Args:
+        strategy_spec: Strategy string like:
+            - "deduce:comment"
+            - "deduce:[comment, notes]"
+    
+    Returns:
+        Tuple of (strategy_type, source_columns)
+        - strategy_type: "deduce"
+        - source_columns: List of source column names
+    
+    Raises:
+        ValidationError: If strategy format is invalid
+    
+    Examples:
+        >>> parse_deduce_strategy("deduce:comment")
+        ("deduce", ["comment"])
+        
+        >>> parse_deduce_strategy("deduce:[comment, notes]")
+        ("deduce", ["comment", "notes"])
+    """
+    if not strategy_spec.startswith("deduce:"):
+        raise ValidationError(
+            f"Invalid deduce strategy: {strategy_spec}. "
+            "Must start with 'deduce:'"
+        )
+    
+    # Extract source specification after "deduce:"
+    source_spec = strategy_spec[7:].strip()  # Remove "deduce:" prefix
+    
+    if not source_spec:
+        raise ValidationError(
+            f"Deduce strategy requires source column(s): {strategy_spec}. "
+            "Format: 'deduce:column' or 'deduce:[col1, col2]'"
+        )
+    
+    # Check if it's multiple columns: deduce:[col1, col2]
+    if source_spec.startswith("[") and source_spec.endswith("]"):
+        # Multiple columns
+        columns_str = source_spec[1:-1]  # Remove brackets
+        
+        if not columns_str.strip():
+            raise ValidationError(
+                f"Deduce column list cannot be empty: {strategy_spec}"
+            )
+        
+        # Split by comma and strip whitespace
+        columns = [c.strip() for c in columns_str.split(",")]
+        
+        if len(columns) == 0:
+            raise ValidationError(
+                f"Deduce strategy must specify at least one column: {strategy_spec}"
+            )
+        
+        return "deduce", columns
+    else:
+        # Single column
+        return "deduce", [source_spec]
+
+
+def is_deduce_strategy(strategy_spec: str) -> bool:
+    """
+    Check if a strategy specification is a deduce strategy.
+    
+    Args:
+        strategy_spec: Strategy string
+    
+    Returns:
+        True if it's a deduce strategy, False otherwise
+    """
+    return isinstance(strategy_spec, str) and strategy_spec.startswith("deduce:")

additory/{augment/augmentor.py → synthetic/synthesizer.py}

@@ -18,7 +18,7 @@ from additory.common.backend import detect_backend, to_polars, from_polars
 from additory.common.exceptions import ValidationError, AugmentError
 from additory.common.validation import validate_dataframe
 from additory.common.sample_data import get_sample_dataset
-from additory.augment.strategies import (
+from additory.synthetic.strategies import (
     parse_strategy_dict,
     get_column_strategy,
     apply_increment_strategy,
@@ -27,6 +27,14 @@ from additory.augment.strategies import (
     parse_strategy_params
 )
 
+# Linked lists feature imports
+from additory.synthetic.namespace_lookup import lookup_linked_list
+from additory.synthetic.linked_list_parser import (
+    parse_linked_list,
+    generate_linked_list_data
+)
+from additory.synthetic.column_name_resolver import resolve_column_names
+
 
 def _validate_generative_strategies(strategy_dict: Dict[str, str]) -> None:
     """
@@ -36,7 +44,7 @@ def _validate_generative_strategies(strategy_dict: Dict[str, str]) -> None:
     - increment (with start parameter)
     - range
     - choice
-    - choice_list
+    - lists (inline linked lists)
     
     Augmentative strategies require existing data:
     - auto (random sampling)
@@ -61,6 +69,10 @@ def _validate_generative_strategies(strategy_dict: Dict[str, str]) -> None:
         # Get the base strategy name (before any parameters)
         strategy_name = strategy_spec.split(":")[0].strip()
         
+        # Handle lists@ pattern
+        if strategy_name.startswith("lists@"):
+            continue  # Valid generative strategy
+        
         if strategy_name in augmentative_strategies:
             invalid_columns.append((col, strategy_name))
     
@@ -76,7 +88,7 @@ def _validate_generative_strategies(strategy_dict: Dict[str, str]) -> None:
         error_lines.append("  - increment (with start parameter)")
         error_lines.append("  - range:min-max")
         error_lines.append("  - choice:[value1,value2,...]")
-        error_lines.append("  - choice_list:list_name")
+        error_lines.append("  - lists@variable_name (inline linked lists)")
         
         raise ValidationError("\n".join(error_lines))
 
@@ -249,7 +261,7 @@ def _augment_polars_engine(df_polars: Any, n_rows: int, strategy_dict: Dict[str,
                 new_data[col] = new_values
             elif col_strategy.startswith("forecast"):
                 # Import here to avoid circular dependency
-                from additory.augment.strategies import apply_forecast_strategy
+                from additory.synthetic.strategies import apply_forecast_strategy
                 
                 # Generate forecasted values
                 new_values = apply_forecast_strategy(
@@ -265,7 +277,7 @@ def _augment_polars_engine(df_polars: Any, n_rows: int, strategy_dict: Dict[str,
                 new_data[col] = new_values
             elif col_strategy.startswith(("normal", "uniform", "skewed_left", "skewed_right", "beta", "gamma", "exponential", "kde")):
                 # Import here to avoid circular dependency
-                from additory.augment.strategies import apply_distribution_strategy
+                from additory.synthetic.strategies import apply_distribution_strategy
                 
                 # Generate distribution values
                 new_values = apply_distribution_strategy(
@@ -310,7 +322,6 @@ def _create_from_scratch_engine(
     - increment (with start parameter)
     - range
     - choice
-    - choice_list
     
     Augmentative strategies (NOT supported):
     - auto (requires existing data)
@@ -349,23 +360,62 @@ def _create_from_scratch_engine(
         ...         "id": "increment:start=1",
         ...         "emp_id": "increment:start=1:pattern=EMP_[001]",
         ...         "age": "range:18-65",
-        ...         "status": "choice:[Active,Inactive]",
-        ...         "department": "choice_list:departments"
+        ...         "status": "choice:[Active,Inactive,Pending]"
         ...     },
         ...     seed=42
         ... )
         >>> result.shape
-        (100, 5)
+        (100, 4)
     """
     import polars as pl
     
     # Validate all strategies are generative
     _validate_generative_strategies(strategy_dict)
     
+    # Pre-process linked lists strategies
+    # Linked lists generate multiple columns, so we need to expand strategy_dict
+    expanded_strategy_dict = {}
+    lists_to_process = []  # Store (original_key, var_name, parsed_data, column_names)
+    
+    for col, col_strategy in strategy_dict.items():
+        if col == "__default__":
+            continue
+        
+        # Check for lists@ pattern
+        if col_strategy.startswith("lists@"):
+            # Extract variable name
+            var_name = col_strategy[6:].strip()  # Remove "lists@" prefix
+            
+            try:
+                # Lookup variable in namespace
+                # Depth=5: user -> add.synthetic (API) -> synthetic() -> _create_from_scratch_engine -> here
+                linked_list_data = lookup_linked_list(var_name, depth=5)
+                
+                # Parse linked list
+                parsed_data = parse_linked_list(linked_list_data)
+                
+                # Resolve column names
+                column_names = resolve_column_names(
+                    list_name=var_name,
+                    strategy_key=col,
+                    num_columns=parsed_data['num_columns'],
+                    explicit_names=parsed_data['column_names']
+                )
+                
+                # Store for later processing
+                lists_to_process.append((col, var_name, parsed_data, column_names))
+                
+            except ValidationError as e:
+                raise ValidationError(f"Linked list error for column '{col}': {e}")
+        else:
+            # Regular strategy - keep as is
+            expanded_strategy_dict[col] = col_strategy
+    
     # Build data column by column
     new_data = {}
     
-    for col, col_strategy in strategy_dict.items():
+    # Process regular strategies first
+    for col, col_strategy in expanded_strategy_dict.items():
         if col == "__default__":
             continue
             
@@ -414,13 +464,24 @@ def _create_from_scratch_engine(
                 f"Unknown or unsupported strategy for column '{col}': '{col_strategy}'"
             )
     
+    # Process linked lists strategies
+    for original_key, var_name, parsed_data, column_names in lists_to_process:
+        # Generate data rows
+        data_rows = generate_linked_list_data(parsed_data, n_rows, seed)
+        
+        # Transpose: list of tuples -> dict of lists
+        # data_rows = [(val1_col1, val1_col2), (val2_col1, val2_col2), ...]
+        # -> {col1: [val1_col1, val2_col1, ...], col2: [val1_col2, val2_col2, ...]}
+        for col_idx, col_name in enumerate(column_names):
+            new_data[col_name] = [row[col_idx] for row in data_rows]
+    
     # Build Polars DataFrame from generated columns
     result = pl.DataFrame(new_data)
     
     return result
 
 
-def augment(
+def synthetic(
     df: Any,
     n_rows: Union[int, str] = 5,
     strategy: Union[str, Dict[str, str]] = "auto",
@@ -428,12 +489,12 @@ def augment(
     output_format: str = "pandas"
 ) -> Any:
     """
-    Augment a dataframe by adding synthetic rows based on existing data.
+    Generate synthetic data by extending a dataframe or creating from scratch.
     
     Uses Polars-only architecture:
     1. Detect input format (pandas/polars/cuDF)
     2. Convert to Polars via Arrow bridge (if needed)
-    3. Process augmentation in Polars
+    3. Process synthetic data generation in Polars
     4. Convert back to original format via Arrow bridge
     
     This function adds new rows to a dataframe using various strategies:
@@ -441,7 +502,7 @@ def augment(
     - "increment": Increment numeric or pattern-based values
     - "range:min-max": Random integers within range
     - "choice:[...]": Random selection from inline list
-    - "choice_list:name": Random selection from registered/built-in list
+    - "lists@variable_name": Inline linked lists (generates multiple columns)
     - "forecast:method": Time series forecasting (linear, polynomial, exponential, seasonal)
     - "normal": Normal distribution generation
     - "uniform": Uniform distribution generation
@@ -465,7 +526,6 @@ def augment(
                     "emp_id": "increment:EMP_[001]_ID",
                     "age": "range:18-65",
                     "status": "choice:[Active,Inactive,Pending]",
-                    "bank": "choice_list:banks",
                     "sales": "forecast:seasonal:period=12",
                     "score": "normal:mean=75:std=10",
                     "income": "skewed_right:skewness=1.5"

additory/utilities/units.py

@@ -255,7 +255,10 @@ class UnitConverter:
     """Unit conversion system with Polars processing"""
     
     def __init__(self):
-        self.arrow_bridge = EnhancedArrowBridge()
+        try:
+            self.arrow_bridge = EnhancedArrowBridge()
+        except ArrowBridgeError:
+            self.arrow_bridge = None
         self.conversion_stats = {
             "total_conversions": 0,
             "successful_conversions": 0,

{additory-0.1.0a2.dist-info → additory-0.1.0a4.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: additory
-Version: 0.1.0a2
-Summary: A semantic, extensible dataframe transformation engine with expressions, lookup, synthetic data, and sample-data support.
+Version: 0.1.0a4
+Summary: A semantic, extensible dataframe transformation engine with expressions, lookup, and synthetic data generation support.
 Author: Krishnamoorthy Sankaran
 License: MIT
 Project-URL: homepage, https://github.com/sekarkrishna/additory
@@ -13,6 +13,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pandas>=1.5
 Requires-Dist: polars>=0.20
+Requires-Dist: pyarrow>=10.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: requests>=2.31
 Requires-Dist: toml>=0.10
@@ -34,11 +35,11 @@ Dynamic: license-file
 
 # Additory
 
-**A semantic, extensible dataframe transformation engine with expressions, lookup, synthetic data, and sample-data support.**
+**A semantic, extensible dataframe transformation engine with expressions, lookup, and augmentation support.**
 
 [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![Version](https://img.shields.io/badge/version-0.1.0a1-orange.svg)](https://github.com/sekarkrishna/additory/tree/main/V0.1.0a1/)
+[![Version](https://img.shields.io/badge/version-0.1.0a4-orange.svg)](https://github.com/sekarkrishna/additory)
 
 **Author:** Krishnamoorthy Sankaran
 
@@ -51,17 +52,17 @@ Dynamic: license-file
 ## 📦 Installation
 
 ```bash
-pip install additory==0.1.0a1
+pip install additory==0.1.0a4
 ```
 
 **Optional GPU support:**
 ```bash
-pip install additory[gpu]==0.1.0a1  # Includes cuDF for GPU acceleration
+pip install additory[gpu]==0.1.0a4  # Includes cuDF for GPU acceleration
 ```
 
 **Development installation:**
 ```bash
-pip install additory[dev]==0.1.0a1  # Includes testing and development tools
+pip install additory[dev]==0.1.0a4  # Includes testing and development tools
 ```
 
 ## 🎯 Core Functions
@@ -69,8 +70,8 @@ pip install additory[dev]==0.1.0a1  # Includes testing and development tools
 | Function | Purpose | Example |
 |----------|---------|---------|
 | `add.to()` | Lookup/join operations | `add.to(df1, from_df=df2, bring='col', against='key')` |
-| `add.augment()` | Generate additional data | `add.augment(df, n_rows=1000)` |
-| `add.synth()` | Synthetic data from schemas | `add.synth("schema.toml", rows=5000)` |
+| `add.synthetic()` | Generate additional data | `add.synthetic(df, n_rows=1000)` |
+| `add.deduce()` | Text-based label deduction | `add.deduce(df, from_column='text', to_column='label')` |
 | `add.scan()` | Data profiling & analysis | `add.scan(df, preset="full")` |
 
 ## 🧬 Available Expressions
@@ -119,7 +120,7 @@ import additory as add
 
 # Works with polars
 df_polars = pl.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
-result = add.augment(df_polars, n_rows=100)
+result = add.synthetic(df_polars, n_rows=100)
 
 # Automatic type detection and conversion
 ```
@@ -193,27 +194,42 @@ patients_with_bsa = add.bsa(patients)
 result = add.fitness_score(add.bmr(add.bmi(patients)))
 ```
 
-### 🔄 Augment and Synthetic Data
+### 🔄 Synthetic Data Generation
 
-**Augment** generates more data similar to your existing dataset, while **Synthetic** creates entirely new datasets from schema definitions.
-
-**Key Differences:**
-- **Augment**: Learns patterns from existing data to create similar rows
-- **Synthetic**: Uses predefined schemas to generate structured data
+**Synthetic** generates additional data similar to your existing dataset using inline strategies.
 
 ```python
-# Augment existing data (learns from patterns)
-more_customers = add.augment(customers, n_rows=1000)
+# Extend existing data (learns from patterns)
+more_customers = add.synthetic(customers, n_rows=1000)
 
 # Create data from scratch with strategies
-new_data = add.augment("@new", n_rows=500, strategy={
+new_data = add.synthetic("@new", n_rows=500, strategy={
     'id': 'increment:start=1',
     'name': 'choice:[John,Jane,Bob]',
     'age': 'range:18-65'
 })
+```
+
+### 🤖 Text-Based Label Deduction
 
-# Generate from schema file (structured approach)
-customers = add.synth("customer_schema.toml", rows=10000)
+**Deduce** automatically fills in missing labels by learning from your existing labeled examples. Pure Python, no LLMs, offline-first.
+
+```python
+# Deduce missing labels from text
+tickets = pd.DataFrame({
+    "ticket_text": ["Cannot log in", "Billing question", "App crashes", "Need invoice"],
+    "category": ["Technical", "Billing", None, None]
+})
+
+# Automatically fill in missing categories
+result = add.deduce(tickets, from_column="ticket_text", to_column="category")
+
+# Use multiple columns for better accuracy
+result = add.deduce(
+    df,
+    from_column=["title", "description"],
+    to_column="category"
+)
 ```
 
 ## 🧪 Examples
@@ -231,7 +247,7 @@ customers = pd.DataFrame({
 })
 
 # Generate more customers
-customers = add.augment(customers, n_rows=10000)
+customers = add.synthetic(customers, n_rows=10000)
 
 # Add customer tiers
 tiers = pd.DataFrame({
@@ -257,7 +273,7 @@ strategy = {
     'height_cm': 'range:150-200'  # Height in cm
 }
 
-patients = add.augment("@new", n_rows=1000, strategy=strategy)
+patients = add.synthetic("@new", n_rows=1000, strategy=strategy)
 
 # Convert height to meters for expressions
 patients['height_m'] = patients['height_cm'] / 100
@@ -272,19 +288,19 @@ print(result.correlations)
 
 ## 📚 Documentation
 
-- **[Function Documentation](https://github.com/sekarkrishna/additory/tree/main/V0.1.0a1/documentation/V0.1.0/)** - Detailed guides for each function
-- **[Expressions Guide](https://github.com/sekarkrishna/additory/tree/main/V0.1.0a1/documentation/V0.1.0/expressions.html)** - Complete expressions reference
+- **[Function Documentation](https://github.com/sekarkrishna/additory/tree/main/documentation/)** - Detailed guides for each function
+- **[Expressions Guide](https://github.com/sekarkrishna/additory/tree/main/documentation/)** - Complete expressions reference
 
 ## 📄 License
 
-MIT License - see [LICENSE](https://github.com/sekarkrishna/additory/tree/main/V0.1.0a1/LICENSE) file for details.
+MIT License - see [LICENSE](LICENSE) file for details.
 
 ## 📞 Support
 
 - **Issues**: [GitHub Issues](https://github.com/sekarkrishna/additory/issues)
-- **Documentation**: [Full Documentation](https://github.com/sekarkrishna/additory/tree/main/V0.1.0a1/documentation/V0.1.0)
+- **Documentation**: [Full Documentation](https://github.com/sekarkrishna/additory/tree/main/documentation/)
 
-## 🗺️ v0.1.1 (February 2025)
+## 🗺️ v0.1.1 (January 2026)
 - Enhanced documentation and tutorials
 - Performance optimizations
 - Additional expressions