additory 0.1.0a2__py3-none-any.whl → 0.1.0a3__py3-none-any.whl

additory/synthetic/namespace_lookup.py

@@ -0,0 +1,129 @@
+"""
+Namespace Lookup for Linked Lists
+
+Finds Python variables in the caller's namespace using frame inspection.
+"""
+
+import inspect
+from typing import Any
+
+from additory.common.exceptions import ValidationError
+
+
+def lookup_variable_in_namespace(var_name: str, depth: int = 2) -> Any:
+    """
+    Look up a variable in the caller's namespace.
+    
+    Uses frame inspection to find variables defined in the same scope
+    as the synthetic() call.
+    
+    Args:
+        var_name: Name of the variable to find
+        depth: Number of frames to go back (default: 2)
+               2 = caller's caller (synthetic() -> this function -> caller)
+        
+    Returns:
+        Value of the variable
+        
+    Raises:
+        ValidationError: If variable not found
+        
+    Examples:
+        >>> # In user code:
+        >>> AE_CM = [["Headache", ["Aspirin"]]]
+        >>> df = add.synthetic('@new', strategy={'col1': 'lists@AE_CM'})
+        >>> # lookup_variable_in_namespace('AE_CM') finds the list
+    """
+    try:
+        # Get caller's frame
+        frame = inspect.currentframe()
+        
+        # Go back 'depth' frames
+        for _ in range(depth):
+            if frame is None:
+                raise ValidationError(
+                    f"Cannot access caller's namespace (frame depth {depth})"
+                )
+            frame = frame.f_back
+        
+        if frame is None:
+            raise ValidationError(
+                f"Cannot access caller's namespace (frame is None)"
+            )
+        
+        # Search in locals first, then globals
+        caller_locals = frame.f_locals
+        caller_globals = frame.f_globals
+        
+        if var_name in caller_locals:
+            return caller_locals[var_name]
+        elif var_name in caller_globals:
+            return caller_globals[var_name]
+        else:
+            # Variable not found - provide helpful error
+            raise ValidationError(
+                f"Variable '{var_name}' not found in namespace.\n"
+                f"Make sure '{var_name}' is defined before calling synthetic().\n"
+                f"\n"
+                f"Example:\n"
+                f"  {var_name} = [['Headache', ['Aspirin'], ['mild']]]\n"
+                f"  df = add.synthetic('@new', strategy={{'col1': 'lists@{var_name}'}})\n"
+                f"\n"
+                f"Note: Linked lists must be defined in the same scope (cell/function) "
+                f"as the synthetic() call."
+            )
+    
+    finally:
+        # Clean up frame reference to avoid reference cycles
+        del frame
+
+
+def validate_linked_list_variable(var_value: Any, var_name: str) -> None:
+    """
+    Validate that the variable is a valid linked list.
+    
+    Args:
+        var_value: Value of the variable
+        var_name: Name of the variable (for error messages)
+        
+    Raises:
+        ValidationError: If variable is not a valid linked list
+    """
+    if not isinstance(var_value, list):
+        raise ValidationError(
+            f"Variable '{var_name}' must be a list. "
+            f"Got: {type(var_value).__name__}\n"
+            f"\n"
+            f"Expected format:\n"
+            f"  {var_name} = [\n"
+            f"      ['Headache', ['Aspirin', 'Ibuprofen'], ['mild', 'moderate']],\n"
+            f"      ['Nausea', ['Ondansetron'], ['severe']]\n"
+            f"  ]"
+        )
+    
+    if len(var_value) == 0:
+        raise ValidationError(
+            f"Variable '{var_name}' is an empty list. "
+            f"Linked list must contain at least one row."
+        )
+
+
+def lookup_linked_list(var_name: str, depth: int = 2) -> Any:
+    """
+    Look up and validate a linked list variable.
+    
+    Convenience function that combines lookup and validation.
+    
+    Args:
+        var_name: Name of the variable to find
+        depth: Number of frames to go back
+        
+    Returns:
+        Linked list data
+        
+    Raises:
+        ValidationError: If variable not found or invalid
+    """
+    var_value = lookup_variable_in_namespace(var_name, depth)
+    validate_linked_list_variable(var_value, var_name)
+    return var_value

additory/{augment → synthetic}/smote.py

@@ -1,5 +1,5 @@
 """
-SMOTE (Synthetic Minority Over-sampling Technique) for Data Augmentation
+SMOTE (Synthetic Minority Over-sampling Technique) for Synthetic Data Generation
 
 Provides imbalanced data handling strategies:
 - SMOTE: Generate synthetic samples for minority class

additory/{augment → synthetic}/strategies.py

@@ -1,11 +1,10 @@
 """
-Strategy handlers for data augmentation
+Strategy handlers for synthetic data generation
 
 Provides different strategies for generating synthetic data:
 - auto: Random sampling from existing values
 - increment: Increment numeric or pattern-based values
 - choice:[...]: Random selection from inline list
-- choice_list:name: Random selection from registered/built-in list
 """
 
 import re
@@ -13,7 +12,6 @@ import random
 from typing import Any, Dict, List, Optional, Tuple
 
 from additory.common.exceptions import ValidationError, AugmentError
-from additory.augment.list_registry import get_list
 
 
 def parse_strategy_params(strategy_spec: str) -> Tuple[str, Dict[str, Any]]:
@@ -319,7 +317,7 @@ def apply_increment_strategy(
     Apply increment strategy to a column (Polars-only).
     
     Supports two modes:
-    1. Augment mode: Increment from last value in df_polars
+    1. Extend mode: Increment from last value in df_polars
     2. Create mode: Start from specified value (requires params with 'start')
     
     Args:
@@ -336,7 +334,7 @@ def apply_increment_strategy(
         ValidationError: If strategy cannot be applied
     
     Examples:
-        # Augment mode (with DataFrame)
+        # Extend mode (with DataFrame)
         >>> apply_increment_strategy(df, "id", "increment", 5)
         [11, 12, 13, 14, 15]  # if last value was 10
         
@@ -349,7 +347,7 @@ def apply_increment_strategy(
         ...                          {"start": 1, "pattern": "EMP_[001]"})
         ["EMP_001", "EMP_002", "EMP_003"]
     """
-    # Determine mode: augment (has df) or create (no df)
+    # Determine mode: extend (has df) or create (no df)
     is_create_mode = df_polars is None
     
     if is_create_mode:
@@ -402,7 +400,7 @@ def apply_increment_strategy(
         return new_values
     
     else:
-        # Augment mode: use existing logic
+        # Extend mode: use existing logic
         # Parse the strategy
         pattern, regex_pattern = parse_increment_strategy(strategy_spec)
         
@@ -483,12 +481,11 @@ def parse_choice_strategy(strategy_spec: str) -> Tuple[str, Optional[List[Any]]]
     Args:
         strategy_spec: Strategy string like:
             - "choice:[value1,value2,value3]"
-            - "choice_list:banks"
     
     Returns:
         Tuple of (strategy_type, values)
-        - strategy_type: "choice" or "choice_list"
-        - values: List of values (for choice) or None (for choice_list)
+        - strategy_type: "choice"
+        - values: List of values
     
     Raises:
         ValidationError: If strategy format is invalid
@@ -496,9 +493,6 @@ def parse_choice_strategy(strategy_spec: str) -> Tuple[str, Optional[List[Any]]]
     Examples:
         >>> parse_choice_strategy("choice:[Active,Inactive,Pending]")
         ("choice", ["Active", "Inactive", "Pending"])
-        
-        >>> parse_choice_strategy("choice_list:banks")
-        ("choice_list", None)
     """
     if strategy_spec.startswith("choice:["):
         # Inline list: choice:[value1,value2,value3]
@@ -526,22 +520,10 @@ def parse_choice_strategy(strategy_spec: str) -> Tuple[str, Optional[List[Any]]]
         
         return "choice", values
     
-    elif strategy_spec.startswith("choice_list:"):
-        # Named list: choice_list:banks
-        list_name = strategy_spec[len("choice_list:"):].strip()
-        
-        if not list_name:
-            raise ValidationError(
-                f"Invalid choice_list strategy: {strategy_spec}. "
-                "Must be in format: choice_list:list_name"
-            )
-        
-        return "choice_list", list_name
-    
     else:
         raise ValidationError(
             f"Invalid choice strategy: {strategy_spec}. "
-            "Must start with 'choice:[' or 'choice_list:'"
+            "Must start with 'choice:['"
         )
 
 
@@ -609,22 +591,7 @@ def apply_choice_strategy(
         ValidationError: If strategy cannot be applied
     """
     # Parse the strategy
-    strategy_type, values_or_name = parse_choice_strategy(strategy_spec)
-    
-    # Get the actual values list
-    if strategy_type == "choice":
-        values = values_or_name
-    elif strategy_type == "choice_list":
-        # Resolve list name to actual list
-        list_name = values_or_name
-        try:
-            values = get_list(list_name)
-        except ValidationError as e:
-            raise ValidationError(
-                f"Cannot apply choice_list strategy: {e}"
-            )
-    else:
-        raise ValidationError(f"Unknown choice strategy type: {strategy_type}")
+    strategy_type, values = parse_choice_strategy(strategy_spec)
     
     # Generate random selections
     if seed is not None:
@@ -675,7 +642,7 @@ def apply_forecast_strategy(
         >>> apply_forecast_strategy(df, "sales", "forecast:seasonal:period=12", 24)
         [98.5, 102.3, 95.8, ...]
     """
-    from additory.augment.forecast import forecast_values, ForecastMethod
+    from additory.synthetic.forecast import forecast_values, ForecastMethod
     
     # Parse strategy: forecast:method:param1=val1:param2=val2
     parts = strategy_spec.split(":")
@@ -845,7 +812,7 @@ def apply_smote_strategy(
         >>> apply_smote_strategy(df, ["feature1", "feature2"], "smote:k=5", 100)
         {"feature1": [1.2, 3.4, ...], "feature2": [5.6, 7.8, ...]}
     """
-    from additory.augment.smote import generate_smote_values
+    from additory.synthetic.smote import generate_smote_values
     
     # Parse strategy: smote:k=5
     parts = strategy_spec.split(":")

additory/{augment/augmentor.py → synthetic/synthesizer.py}

@@ -18,7 +18,7 @@ from additory.common.backend import detect_backend, to_polars, from_polars
 from additory.common.exceptions import ValidationError, AugmentError
 from additory.common.validation import validate_dataframe
 from additory.common.sample_data import get_sample_dataset
-from additory.augment.strategies import (
+from additory.synthetic.strategies import (
     parse_strategy_dict,
     get_column_strategy,
     apply_increment_strategy,
@@ -27,6 +27,14 @@ from additory.augment.strategies import (
     parse_strategy_params
 )
 
+# Linked lists feature imports
+from additory.synthetic.namespace_lookup import lookup_linked_list
+from additory.synthetic.linked_list_parser import (
+    parse_linked_list,
+    generate_linked_list_data
+)
+from additory.synthetic.column_name_resolver import resolve_column_names
+
 
 def _validate_generative_strategies(strategy_dict: Dict[str, str]) -> None:
     """
@@ -36,7 +44,7 @@ def _validate_generative_strategies(strategy_dict: Dict[str, str]) -> None:
     - increment (with start parameter)
     - range
     - choice
-    - choice_list
+    - lists (inline linked lists)
     
     Augmentative strategies require existing data:
     - auto (random sampling)
@@ -61,6 +69,10 @@ def _validate_generative_strategies(strategy_dict: Dict[str, str]) -> None:
         # Get the base strategy name (before any parameters)
         strategy_name = strategy_spec.split(":")[0].strip()
         
+        # Handle lists@ pattern
+        if strategy_name.startswith("lists@"):
+            continue  # Valid generative strategy
+        
         if strategy_name in augmentative_strategies:
             invalid_columns.append((col, strategy_name))
     
@@ -76,7 +88,7 @@ def _validate_generative_strategies(strategy_dict: Dict[str, str]) -> None:
         error_lines.append("  - increment (with start parameter)")
         error_lines.append("  - range:min-max")
         error_lines.append("  - choice:[value1,value2,...]")
-        error_lines.append("  - choice_list:list_name")
+        error_lines.append("  - lists@variable_name (inline linked lists)")
         
         raise ValidationError("\n".join(error_lines))
 
@@ -249,7 +261,7 @@ def _augment_polars_engine(df_polars: Any, n_rows: int, strategy_dict: Dict[str,
                 new_data[col] = new_values
             elif col_strategy.startswith("forecast"):
                 # Import here to avoid circular dependency
-                from additory.augment.strategies import apply_forecast_strategy
+                from additory.synthetic.strategies import apply_forecast_strategy
                 
                 # Generate forecasted values
                 new_values = apply_forecast_strategy(
@@ -265,7 +277,7 @@ def _augment_polars_engine(df_polars: Any, n_rows: int, strategy_dict: Dict[str,
                 new_data[col] = new_values
             elif col_strategy.startswith(("normal", "uniform", "skewed_left", "skewed_right", "beta", "gamma", "exponential", "kde")):
                 # Import here to avoid circular dependency
-                from additory.augment.strategies import apply_distribution_strategy
+                from additory.synthetic.strategies import apply_distribution_strategy
                 
                 # Generate distribution values
                 new_values = apply_distribution_strategy(
@@ -310,7 +322,6 @@ def _create_from_scratch_engine(
     - increment (with start parameter)
     - range
     - choice
-    - choice_list
     
     Augmentative strategies (NOT supported):
     - auto (requires existing data)
@@ -349,23 +360,62 @@ def _create_from_scratch_engine(
         ...         "id": "increment:start=1",
         ...         "emp_id": "increment:start=1:pattern=EMP_[001]",
         ...         "age": "range:18-65",
-        ...         "status": "choice:[Active,Inactive]",
-        ...         "department": "choice_list:departments"
+        ...         "status": "choice:[Active,Inactive,Pending]"
         ...     },
         ...     seed=42
         ... )
         >>> result.shape
-        (100, 5)
+        (100, 4)
     """
     import polars as pl
     
     # Validate all strategies are generative
     _validate_generative_strategies(strategy_dict)
     
+    # Pre-process linked lists strategies
+    # Linked lists generate multiple columns, so we need to expand strategy_dict
+    expanded_strategy_dict = {}
+    lists_to_process = []  # Store (original_key, var_name, parsed_data, column_names)
+    
+    for col, col_strategy in strategy_dict.items():
+        if col == "__default__":
+            continue
+        
+        # Check for lists@ pattern
+        if col_strategy.startswith("lists@"):
+            # Extract variable name
+            var_name = col_strategy[6:].strip()  # Remove "lists@" prefix
+            
+            try:
+                # Lookup variable in namespace
+                # Depth=5: user -> add.synthetic (API) -> synthetic() -> _create_from_scratch_engine -> here
+                linked_list_data = lookup_linked_list(var_name, depth=5)
+                
+                # Parse linked list
+                parsed_data = parse_linked_list(linked_list_data)
+                
+                # Resolve column names
+                column_names = resolve_column_names(
+                    list_name=var_name,
+                    strategy_key=col,
+                    num_columns=parsed_data['num_columns'],
+                    explicit_names=parsed_data['column_names']
+                )
+                
+                # Store for later processing
+                lists_to_process.append((col, var_name, parsed_data, column_names))
+                
+            except ValidationError as e:
+                raise ValidationError(f"Linked list error for column '{col}': {e}")
+        else:
+            # Regular strategy - keep as is
+            expanded_strategy_dict[col] = col_strategy
+    
     # Build data column by column
     new_data = {}
     
-    for col, col_strategy in strategy_dict.items():
+    # Process regular strategies first
+    for col, col_strategy in expanded_strategy_dict.items():
         if col == "__default__":
             continue
             
@@ -414,13 +464,24 @@ def _create_from_scratch_engine(
                 f"Unknown or unsupported strategy for column '{col}': '{col_strategy}'"
             )
     
+    # Process linked lists strategies
+    for original_key, var_name, parsed_data, column_names in lists_to_process:
+        # Generate data rows
+        data_rows = generate_linked_list_data(parsed_data, n_rows, seed)
+        
+        # Transpose: list of tuples -> dict of lists
+        # data_rows = [(val1_col1, val1_col2), (val2_col1, val2_col2), ...]
+        # -> {col1: [val1_col1, val2_col1, ...], col2: [val1_col2, val2_col2, ...]}
+        for col_idx, col_name in enumerate(column_names):
+            new_data[col_name] = [row[col_idx] for row in data_rows]
+    
     # Build Polars DataFrame from generated columns
     result = pl.DataFrame(new_data)
     
     return result
 
 
-def augment(
+def synthetic(
     df: Any,
     n_rows: Union[int, str] = 5,
     strategy: Union[str, Dict[str, str]] = "auto",
@@ -428,12 +489,12 @@ def augment(
     output_format: str = "pandas"
 ) -> Any:
     """
-    Augment a dataframe by adding synthetic rows based on existing data.
+    Generate synthetic data by extending a dataframe or creating from scratch.
     
     Uses Polars-only architecture:
     1. Detect input format (pandas/polars/cuDF)
     2. Convert to Polars via Arrow bridge (if needed)
-    3. Process augmentation in Polars
+    3. Process synthetic data generation in Polars
     4. Convert back to original format via Arrow bridge
     
     This function adds new rows to a dataframe using various strategies:
@@ -441,7 +502,7 @@ def augment(
     - "increment": Increment numeric or pattern-based values
     - "range:min-max": Random integers within range
     - "choice:[...]": Random selection from inline list
-    - "choice_list:name": Random selection from registered/built-in list
+    - "lists@variable_name": Inline linked lists (generates multiple columns)
     - "forecast:method": Time series forecasting (linear, polynomial, exponential, seasonal)
     - "normal": Normal distribution generation
     - "uniform": Uniform distribution generation
@@ -465,7 +526,6 @@ def augment(
                     "emp_id": "increment:EMP_[001]_ID",
                     "age": "range:18-65",
                     "status": "choice:[Active,Inactive,Pending]",
-                    "bank": "choice_list:banks",
                     "sales": "forecast:seasonal:period=12",
                     "score": "normal:mean=75:std=10",
                     "income": "skewed_right:skewness=1.5"

additory/utilities/units.py

@@ -255,7 +255,10 @@ class UnitConverter:
     """Unit conversion system with Polars processing"""
     
     def __init__(self):
-        self.arrow_bridge = EnhancedArrowBridge()
+        try:
+            self.arrow_bridge = EnhancedArrowBridge()
+        except ArrowBridgeError:
+            self.arrow_bridge = None
         self.conversion_stats = {
             "total_conversions": 0,
             "successful_conversions": 0,

{additory-0.1.0a2.dist-info → additory-0.1.0a3.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: additory
-Version: 0.1.0a2
-Summary: A semantic, extensible dataframe transformation engine with expressions, lookup, synthetic data, and sample-data support.
+Version: 0.1.0a3
+Summary: A semantic, extensible dataframe transformation engine with expressions, lookup, and synthetic data generation support.
 Author: Krishnamoorthy Sankaran
 License: MIT
 Project-URL: homepage, https://github.com/sekarkrishna/additory
@@ -13,6 +13,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pandas>=1.5
 Requires-Dist: polars>=0.20
+Requires-Dist: pyarrow>=10.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: requests>=2.31
 Requires-Dist: toml>=0.10
@@ -34,11 +35,11 @@ Dynamic: license-file
 
 # Additory
 
-**A semantic, extensible dataframe transformation engine with expressions, lookup, synthetic data, and sample-data support.**
+**A semantic, extensible dataframe transformation engine with expressions, lookup, and augmentation support.**
 
 [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![Version](https://img.shields.io/badge/version-0.1.0a1-orange.svg)](https://github.com/sekarkrishna/additory/tree/main/V0.1.0a1/)
+[![Version](https://img.shields.io/badge/version-0.1.0a2-orange.svg)](https://github.com/sekarkrishna/additory/tree/main/V0.1.0a1/)
 
 **Author:** Krishnamoorthy Sankaran
 
@@ -51,17 +52,17 @@ Dynamic: license-file
 ## 📦 Installation
 
 ```bash
-pip install additory==0.1.0a1
+pip install additory==0.1.0a2
 ```
 
 **Optional GPU support:**
 ```bash
-pip install additory[gpu]==0.1.0a1  # Includes cuDF for GPU acceleration
+pip install additory[gpu]==0.1.0a2  # Includes cuDF for GPU acceleration
 ```
 
 **Development installation:**
 ```bash
-pip install additory[dev]==0.1.0a1  # Includes testing and development tools
+pip install additory[dev]==0.1.0a2  # Includes testing and development tools
 ```
 
 ## 🎯 Core Functions
@@ -70,7 +71,6 @@ pip install additory[dev]==0.1.0a1  # Includes testing and development tools
 |----------|---------|---------|
 | `add.to()` | Lookup/join operations | `add.to(df1, from_df=df2, bring='col', against='key')` |
 | `add.augment()` | Generate additional data | `add.augment(df, n_rows=1000)` |
-| `add.synth()` | Synthetic data from schemas | `add.synth("schema.toml", rows=5000)` |
 | `add.scan()` | Data profiling & analysis | `add.scan(df, preset="full")` |
 
 ## 🧬 Available Expressions
@@ -193,13 +193,9 @@ patients_with_bsa = add.bsa(patients)
 result = add.fitness_score(add.bmr(add.bmi(patients)))
 ```
 
-### 🔄 Augment and Synthetic Data
+### 🔄 Augment Data Generation
 
-**Augment** generates more data similar to your existing dataset, while **Synthetic** creates entirely new datasets from schema definitions.
-
-**Key Differences:**
-- **Augment**: Learns patterns from existing data to create similar rows
-- **Synthetic**: Uses predefined schemas to generate structured data
+**Augment** generates additional data similar to your existing dataset using inline strategies.
 
 ```python
 # Augment existing data (learns from patterns)
@@ -211,9 +207,6 @@ new_data = add.augment("@new", n_rows=500, strategy={
     'name': 'choice:[John,Jane,Bob]',
     'age': 'range:18-65'
 })
-
-# Generate from schema file (structured approach)
-customers = add.synth("customer_schema.toml", rows=10000)
 ```
 
 ## 🧪 Examples