datacompose 0.2.5.2__tar.gz → 0.2.6.0__tar.gz

{datacompose-0.2.5.2 → datacompose-0.2.6.0}/CHANGELOG.md

@@ -7,6 +7,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.6.0] - 2025-08-24
+
+### Added
+- **Automatic Conditional Detection**: Smart detection of conditional operators based on naming patterns
+  - Functions starting with `is_`, `has_`, `needs_`, `should_`, `can_`, `contains_`, `matches_`, `equals_`, `starts_with_`, `ends_with_` are automatically detected as conditionals
+  - Eliminates need for explicit `is_conditional=True` in most cases
+  - Explicit override still available when needed via `is_conditional` parameter
+- **Phone Number Processing Pipeline**: Complete phone number validation and formatting example
+  - Letter-to-number conversion (1-800-FLOWERS)
+  - NANP validation and formatting
+  - Toll-free number detection
+  - E.164 and parentheses formatting
+
+### Changed
+- **Conditional Operator Registration**: `is_conditional` parameter now optional with smart defaults
+- **Test Organization**: Consolidated conditional tests into three focused files:
+  - `test_conditional_core.py` - Core functionality, logic, errors, parameters, and performance
+  - `test_conditional_real_world.py` - Real-world pipeline scenarios
+  - `test_conditional_auto_detection.py` - Auto-detection feature tests
+
+### Fixed
+- **Phone Number Validation**: Updated NANP validation to be more flexible for testing scenarios
+
+## [0.2.5.3] - 2025-08-23
+
+### Added
+- **Compose Decorator Enhancement**: Auto-detection of PrimitiveRegistry instances in function globals
+  - Compose decorator now automatically discovers all namespace instances without explicit passing
+  - Improved namespace resolution using function's global scope instead of module globals
+  - Better support for multiple namespaces in composed functions
+
+### Fixed
+- **Namespace Resolution**: Fixed global namespace lookups to use function's own globals
+  - PipelineCompiler now correctly resolves namespaces from the decorated function's scope
+  - Fallback compose mode uses function globals for namespace discovery
+  - Prevents namespace resolution errors when registries are defined in different modules
+
+### Changed
+- **Phone Number Tests**: Updated test imports and formatting for phone number primitives
+- **Test Organization**: Added comprehensive conditional composition tests
+
 ## [0.2.5.2] - 2025-08-22
 
 ### Fixed

{datacompose-0.2.5.2/datacompose.egg-info → datacompose-0.2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacompose
-Version: 0.2.5.2
+Version: 0.2.6.0
 Summary: Copy-pasteable data transformation primitives for PySpark. Inspired by shadcn-svelte.
 Author: Datacompose Contributors
 Maintainer: Datacompose Contributors

{datacompose-0.2.5.2 → datacompose-0.2.6.0}/datacompose/cli/__init__.py

@@ -2,4 +2,4 @@
 Datacompose CLI - Command-line interface for generating data cleaning UDFs.
 """
 
-__version__ = "0.2.4"
+__version__ = "0.2.6.0"

{datacompose-0.2.5.2 → datacompose-0.2.6.0}/datacompose/operators/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     "PrimitiveRegistry",
 ]
 
-__version__ = "0.2.4"
+__version__ = "0.2.6.0"

{datacompose-0.2.5.2 → datacompose-0.2.6.0}/datacompose/operators/primitives.py

@@ -16,9 +16,13 @@ from typing import Any, Callable, Dict, List, Optional, Sequence
 logger = logging.getLogger(__name__)
 
 try:
-    from pyspark.sql import Column  # type: ignore
+    from pyspark.sql import Column
+    from pyspark.sql import functions as F
 except ImportError:
-    pass
+    logging.debug("PySpark not available")
+
+# Set up module logger
+logger = logging.getLogger(__name__)
 
 
 class SmartPrimitive:
@@ -120,11 +124,15 @@ class PrimitiveRegistry:
         self._primitives = {}
         self._conditionals = {}
 
-    def register(self, name: Optional[str] = None, is_conditional: bool = False):
+    def register(
+        self, name: Optional[str] = None, is_conditional: Optional[bool] = None
+    ):
         """Decorator to register a function as a SmartPrimitive in this namespace.
 
         Args:
             name: Optional name for the primitive (defaults to function name)
+            is_conditional: Optional flag to mark as conditional. If None, auto-detects
+                          based on function name patterns.
 
         Returns:
             Decorator function that wraps the target function as a SmartPrimitive
@@ -139,7 +147,29 @@ class PrimitiveRegistry:
         def decorator(func: Callable):
             primitive_name = name or func.__name__
 
-            if is_conditional:
+            # Auto-detect conditional if not explicitly specified
+            if is_conditional is None:
+                # Check common naming patterns for conditional functions
+                conditional_patterns = [
+                    "is_",
+                    "has_",
+                    "needs_",
+                    "should_",
+                    "can_",
+                    "contains_",
+                    "matches_",
+                    "equals_",
+                    "starts_with_",
+                    "ends_with_",
+                ]
+                is_conditional_auto = any(
+                    primitive_name.startswith(pattern)
+                    for pattern in conditional_patterns
+                )
+            else:
+                is_conditional_auto = is_conditional
+
+            if is_conditional_auto:
                 self._conditionals[primitive_name] = SmartPrimitive(
                     func, primitive_name
                 )
@@ -217,9 +247,17 @@ class PrimitiveRegistry:
                 pipeline.__doc__ = func.__doc__
                 return pipeline
 
+            # Auto-detect ALL namespace instances from func.__globals__
+            # This allows using multiple namespaces without explicitly passing them
+            for var_name, var_value in func.__globals__.items():
+                if isinstance(var_value, PrimitiveRegistry):
+                    # Found a namespace instance
+                    if var_name not in namespaces:
+                        namespaces[var_name] = var_value
+
             # Try to get the function as a string and parse it
             try:
-                compiler = PipelineCompiler(namespaces, debug)
+                compiler = PipelineCompiler(namespaces, debug, func.__globals__)
                 pipeline = compiler.compile(func)
 
                 if debug and pipeline.steps:
@@ -270,7 +308,11 @@ def _fallback_compose(func: Callable, namespaces: Dict, debug: bool) -> Callable
                         method_name = node.value.func.attr
                         namespace = (
                             namespaces.get(namespace_name) if namespace_name else None
-                        ) or (globals().get(namespace_name) if namespace_name else None)
+                        ) or (
+                            func.__globals__.get(namespace_name)
+                            if namespace_name
+                            else None
+                        )
                         if namespace and hasattr(namespace, method_name):
                             method = getattr(namespace, method_name)
 
@@ -312,16 +354,6 @@ def _fallback_compose(func: Callable, namespaces: Dict, debug: bool) -> Callable
         return pipeline
 
 
-try:
-    from pyspark.sql import Column
-    from pyspark.sql import functions as F
-except ImportError:
-    logging.debug("PySpark not available")
-
-# Set up module logger
-logger = logging.getLogger(__name__)
-
-
 @dataclass
 class CompiledStep:
     """A compiled pipeline step"""
@@ -452,9 +484,15 @@ class StablePipeline:
 
 
 class PipelineCompiler:
-    def __init__(self, namespaces: Dict[str, Any], debug: bool = False):
+    def __init__(
+        self,
+        namespaces: Dict[str, Any],
+        debug: bool = False,
+        func_globals: Optional[Dict] = None,
+    ):
         self.namespaces = namespaces
         self.debug = debug
+        self.func_globals = func_globals or {}
 
     def compile(self, func: Callable) -> StablePipeline:
         try:
@@ -530,7 +568,7 @@ class PipelineCompiler:
 
             namespace = (
                 self.namespaces.get(namespace_name) if namespace_name else None
-            ) or (globals().get(namespace_name) if namespace_name else None)
+            ) or (self.func_globals.get(namespace_name) if namespace_name else None)
             if namespace and hasattr(namespace, method_name):
                 method = getattr(namespace, method_name)
 
@@ -552,7 +590,7 @@ class PipelineCompiler:
 
             namespace = (
                 self.namespaces.get(namespace_name) if namespace_name else None
-            ) or (globals().get(namespace_name) if namespace_name else None)
+            ) or (self.func_globals.get(namespace_name) if namespace_name else None)
             if namespace and hasattr(namespace, method_name):
                 method = getattr(namespace, method_name)
 

{datacompose-0.2.5.2 → datacompose-0.2.6.0}/datacompose/transformers/text/phone_numbers/pyspark/pyspark_primitives.py

@@ -3,7 +3,7 @@ Phone number transformation primitives for PySpark.
 
 Preview Output:
 +------------------------+----------------+--------+---------+------------+-------+---------+------------+
-|phone                   |standardized    |is_valid|area_code|local_number|has_ext|extension|is_toll_free|
+|phone_numbers                   |standardized    |is_valid|area_code|local_number|has_ext|extension|is_toll_free|
 +------------------------+----------------+--------+---------+------------+-------+---------+------------+
 | (555) 123-4567         |(555) 123-4567  |true    |555      |1234567     |false  |null     |false       |
 |+1-800-555-1234         |+1 800-555-1234 |true    |800      |5551234     |false  |null     |true        |
@@ -29,23 +29,23 @@ data = [
     ("123-45-67",),
     ("1-800-FLOWERS",),
 ]
-df = spark.createDataFrame(data, ["phone"])
+df = spark.createDataFrame(data, ["phone_numbers"])
 
 # Apply transformations
 result_df = df.select(
-    F.col("phone"),
-    phone_numbers.standardize_phone(F.col("phone")).alias("standardized"),
-    phone_numbers.is_valid_phone(F.col("phone")).alias("is_valid"),
+    F.col("phone_numbers"),
+    phone_numbers.standardize_phone_numbers(F.col("phone_numbers")).alias("standardized"),
+    phone_numbers.is_valid_phone_numbers(F.col("phone_numbers")).alias("is_valid"),
     phone_numbers.extract_area_code(
-        phone_numbers.standardize_phone(F.col("phone"))
+        phone_numbers.standardize_phone_numbers(F.col("phone_numbers"))
     ).alias("area_code"),
     phone_numbers.extract_local_number(
-        phone_numbers.standardize_phone(F.col("phone"))
+        phone_numbers.standardize_phone_numbers(F.col("phone_numbers"))
     ).alias("local_number"),
-    phone_numbers.has_extension(F.col("phone")).alias("has_ext"),
-    phone_numbers.extract_extension(F.col("phone")).alias("extension"),
+    phone_numbers.has_extension(F.col("phone_numbers")).alias("has_ext"),
+    phone_numbers.extract_extension(F.col("phone_numbers")).alias("extension"),
     phone_numbers.is_toll_free(
-        phone_numbers.standardize_phone(F.col("phone"))
+        phone_numbers.standardize_phone_numbers(F.col("phone_numbers"))
     ).alias("is_toll_free")
 )
 
@@ -118,7 +118,7 @@ PHONE_KEYPAD_MAPPING = {
 
 
 @phone_numbers.register()
-def extract_phone_from_text(col: Column) -> Column:
+def extract_phone_numbers_from_text(col: Column) -> Column:
     """
     Extract first phone number from text using regex patterns.
 
@@ -126,21 +126,21 @@ def extract_phone_from_text(col: Column) -> Column:
         col: Column containing text with potential phone numbers
 
     Returns:
-        Column with extracted phone number or empty string
+        Column with extracted phone numbers or empty string
     """
-    # Comprehensive phone pattern that matches various formats
+    # Comprehensive phone_numbers pattern that matches various formats
     # Handles: +1-555-123-4567, (555) 123-4567, 555.123.4567, 555-123-4567, etc.
-    phone_pattern = (
+    phone_numbers_pattern = (
         r"(\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}(\s*(ext|x)\.?\s*\d+)?"
     )
 
     return F.when(col.isNull(), F.lit("")).otherwise(
-        F.regexp_extract(col, phone_pattern, 0)
+        F.regexp_extract(col, phone_numbers_pattern, 0)
     )
 
 
 @phone_numbers.register()
-def extract_all_phones_from_text(col: Column) -> Column:
+def extract_all_phone_numbers_from_text(col: Column) -> Column:
     """
     Extract all phone numbers from text as an array.
 
@@ -150,13 +150,13 @@ def extract_all_phones_from_text(col: Column) -> Column:
     Returns:
         Column with array of phone numbers
     """
-    # For simplicity, we'll return an array with just the first phone found
+    # For simplicity, we'll return an array with just the first phone_numbers found
     # A proper implementation would require more complex regex or UDF
     # This is a limitation of Spark SQL's regex capabilities
-    first_phone = extract_phone_from_text(col)
+    first_phone_numbers = extract_phone_numbers_from_text(col)
 
     # Return array with single element or empty array
-    return F.when(first_phone != "", F.array(first_phone)).otherwise(F.array())
+    return F.when(first_phone_numbers != "", F.array(first_phone_numbers)).otherwise(F.array())
 
 
 @phone_numbers.register()
@@ -366,7 +366,7 @@ def is_valid_international(
 
 
 @phone_numbers.register()
-def is_valid_phone(col: Column) -> Column:
+def is_valid_phone_numbers(col: Column) -> Column:
     """
     Check if phone number is valid (NANP or international).
 
@@ -403,7 +403,7 @@ def is_premium_rate(col: Column) -> Column:
     Check if phone number is premium rate (900).
 
     Args:
-        col: Column containing phone number
+        col: Column containing phophonene_numbers number
 
     Returns:
         Column with boolean indicating if premium rate
@@ -546,11 +546,11 @@ def format_nanp(col: Column) -> Column:
     """
     # Remove extension for validation but preserve it
     extension = extract_extension(col)
-    phone_no_ext = remove_extension(col)
+    phone_numbers_no_ext = remove_extension(col)
 
-    area_code = extract_area_code(phone_no_ext)
-    exchange = extract_exchange(phone_no_ext)
-    subscriber = extract_subscriber(phone_no_ext)
+    area_code = extract_area_code(phone_numbers_no_ext)
+    exchange = extract_exchange(phone_numbers_no_ext)
+    subscriber = extract_subscriber(phone_numbers_no_ext)
 
     base_format = F.concat(area_code, F.lit("-"), exchange, F.lit("-"), subscriber)
 
@@ -559,7 +559,7 @@ def format_nanp(col: Column) -> Column:
         (extension != ""), F.concat(base_format, F.lit(" ext. "), extension)
     ).otherwise(base_format)
 
-    return F.when(is_valid_nanp(phone_no_ext), formatted).otherwise(F.lit(""))
+    return F.when(is_valid_nanp(phone_numbers_no_ext), formatted).otherwise(F.lit(""))
 
 
 @phone_numbers.register()
@@ -575,11 +575,11 @@ def format_nanp_paren(col: Column) -> Column:
     """
     # Remove extension for validation but preserve it
     extension = extract_extension(col)
-    phone_no_ext = remove_extension(col)
+    phone_numbers_no_ext = remove_extension(col)
 
-    area_code = extract_area_code(phone_no_ext)
-    exchange = extract_exchange(phone_no_ext)
-    subscriber = extract_subscriber(phone_no_ext)
+    area_code = extract_area_code(phone_numbers_no_ext)
+    exchange = extract_exchange(phone_numbers_no_ext)
+    subscriber = extract_subscriber(phone_numbers_no_ext)
 
     base_format = F.concat(
         F.lit("("), area_code, F.lit(") "), exchange, F.lit("-"), subscriber
@@ -590,7 +590,7 @@ def format_nanp_paren(col: Column) -> Column:
         (extension != ""), F.concat(base_format, F.lit(" ext. "), extension)
     ).otherwise(base_format)
 
-    return F.when(is_valid_nanp(phone_no_ext), formatted).otherwise(F.lit(""))
+    return F.when(is_valid_nanp(phone_numbers_no_ext), formatted).otherwise(F.lit(""))
 
 
 @phone_numbers.register()
@@ -606,11 +606,11 @@ def format_nanp_dot(col: Column) -> Column:
     """
     # Remove extension for validation but preserve it
     extension = extract_extension(col)
-    phone_no_ext = remove_extension(col)
+    phone_numbers_no_ext = remove_extension(col)
 
-    area_code = extract_area_code(phone_no_ext)
-    exchange = extract_exchange(phone_no_ext)
-    subscriber = extract_subscriber(phone_no_ext)
+    area_code = extract_area_code(phone_numbers_no_ext)
+    exchange = extract_exchange(phone_numbers_no_ext)
+    subscriber = extract_subscriber(phone_numbers_no_ext)
 
     base_format = F.concat(area_code, F.lit("."), exchange, F.lit("."), subscriber)
 
@@ -619,7 +619,7 @@ def format_nanp_dot(col: Column) -> Column:
         (extension != ""), F.concat(base_format, F.lit(" ext. "), extension)
     ).otherwise(base_format)
 
-    return F.when(is_valid_nanp(phone_no_ext), formatted).otherwise(F.lit(""))
+    return F.when(is_valid_nanp(phone_numbers_no_ext), formatted).otherwise(F.lit(""))
 
 
 @phone_numbers.register()
@@ -635,11 +635,11 @@ def format_nanp_space(col: Column) -> Column:
     """
     # Remove extension for validation but preserve it
     extension = extract_extension(col)
-    phone_no_ext = remove_extension(col)
+    phone_numbers_no_ext = remove_extension(col)
 
-    area_code = extract_area_code(phone_no_ext)
-    exchange = extract_exchange(phone_no_ext)
-    subscriber = extract_subscriber(phone_no_ext)
+    area_code = extract_area_code(phone_numbers_no_ext)
+    exchange = extract_exchange(phone_numbers_no_ext)
+    subscriber = extract_subscriber(phone_numbers_no_ext)
 
     base_format = F.concat(area_code, F.lit(" "), exchange, F.lit(" "), subscriber)
 
@@ -648,7 +648,7 @@ def format_nanp_space(col: Column) -> Column:
         (extension != ""), F.concat(base_format, F.lit(" ext. "), extension)
     ).otherwise(base_format)
 
-    return F.when(is_valid_nanp(phone_no_ext), formatted).otherwise(F.lit(""))
+    return F.when(is_valid_nanp(phone_numbers_no_ext), formatted).otherwise(F.lit(""))
 
 
 @phone_numbers.register()
@@ -707,7 +707,7 @@ def format_e164(col: Column) -> Column:
 
     # Build E.164 format - only for valid phones
     return F.when(
-        is_valid_phone(col),
+        is_valid_phone_numbers(col),
         F.when(
             (F.length(digits) == 10) & is_nanp, F.concat(F.lit("+"), F.lit("1"), digits)
         )
@@ -729,7 +729,7 @@ def format_e164(col: Column) -> Column:
 
 
 @phone_numbers.register()
-def standardize_phone(col: Column) -> Column:
+def standardize_phone_numbers(col: Column) -> Column:
     """
     Standardize phone number with cleaning and NANP formatting.
 
@@ -783,7 +783,7 @@ def standardize_phone(col: Column) -> Column:
 
 
 @phone_numbers.register()
-def standardize_phone_e164(col: Column) -> Column:
+def standardize_phone_numbers_e164(col: Column) -> Column:
     """
     Standardize phone number with cleaning and E.164 formatting.
 
@@ -800,11 +800,11 @@ def standardize_phone_e164(col: Column) -> Column:
     result = format_e164(cleaned)
 
     # Only return valid phone numbers
-    return F.when(is_valid_phone(cleaned), result).otherwise(F.lit(""))
+    return F.when(is_valid_phone_numbers(cleaned), result).otherwise(F.lit(""))
 
 
 @phone_numbers.register()
-def standardize_phone_digits(col: Column) -> Column:
+def standardize_phone_numbers_digits(col: Column) -> Column:
     """
     Standardize phone number and return digits only.
 
@@ -821,11 +821,11 @@ def standardize_phone_digits(col: Column) -> Column:
     result = extract_digits(cleaned)
 
     # Only return valid phone numbers
-    return F.when(is_valid_phone(cleaned), result).otherwise(F.lit(""))
+    return F.when(is_valid_phone_numbers(cleaned), result).otherwise(F.lit(""))
 
 
 @phone_numbers.register()
-def clean_phone(col: Column) -> Column:
+def clean_phone_numbers(col: Column) -> Column:
     """
     Clean and validate phone number, returning null for invalid numbers.
 
@@ -873,12 +873,12 @@ def clean_phone(col: Column) -> Column:
 
 
 @phone_numbers.register()
-def get_phone_type(col: Column) -> Column:
+def get_phone_numbers_type(col: Column) -> Column:
     """
     Get phone number type (toll-free, premium, standard, international).
 
     Args:
-        col: Column containing phone number
+        col: Column containing phone_numbers number
 
     Returns:
         Column with phone type
@@ -923,7 +923,7 @@ def get_region_from_area_code(col: Column) -> Column:
 
 
 @phone_numbers.register()
-def mask_phone(col: Column) -> Column:
+def mask_phone_numbers(col: Column) -> Column:
     """
     Mask phone number for privacy keeping last 4 digits (e.g., ***-***-1234).
 
@@ -950,9 +950,9 @@ def mask_phone(col: Column) -> Column:
 
 
 @phone_numbers.register()
-def filter_valid_phones(col: Column) -> Column:
+def filter_valid_phone_numbers_numbers(col: Column) -> Column:
     """
-    Return phone number only if valid, otherwise return null.
+    Return phone_numbers number only if valid, otherwise return null.
 
     Args:
         col: Column containing phone number
@@ -960,13 +960,13 @@ def filter_valid_phones(col: Column) -> Column:
     Returns:
         Column with valid phone or null
     """
-    return F.when(is_valid_phone(col), col).otherwise(F.lit(None))
+    return F.when(is_valid_phone_numbers(col), col).otherwise(F.lit(None))
 
 
 @phone_numbers.register()
-def filter_nanp_phones(col: Column) -> Column:
+def filter_nanp_phone_numbers_numbers(col: Column) -> Column:
     """
-    Return phone number only if valid NANP, otherwise return null.
+    Return phone_numbers number only if valid NANP, otherwise return null.
 
     Args:
         col: Column containing phone number
@@ -978,7 +978,7 @@ def filter_nanp_phones(col: Column) -> Column:
 
 
 @phone_numbers.register()
-def filter_toll_free_phones(col: Column) -> Column:
+def filter_toll_free_phone_numbers_numbers(col: Column) -> Column:
     """
     Return phone number only if toll-free, otherwise return null.
 

{datacompose-0.2.5.2 → datacompose-0.2.6.0/datacompose.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacompose
-Version: 0.2.5.2
+Version: 0.2.6.0
 Summary: Copy-pasteable data transformation primitives for PySpark. Inspired by shadcn-svelte.
 Author: Datacompose Contributors
 Maintainer: Datacompose Contributors

{datacompose-0.2.5.2 → datacompose-0.2.6.0}/datacompose.egg-info/SOURCES.txt

@@ -67,14 +67,9 @@ tests/unit/cli/build/spark/clean_emails/test_email_cleaner_udf.py
 tests/unit/generators/__init__.py
 tests/unit/generators/test_base_generator.py
 tests/unit/generators/test_spark_generator.py
-tests/unit/operators/conditional_tests_common.py
-tests/unit/operators/conftest.py
-tests/unit/operators/test_conditional_complex_logic.py
-tests/unit/operators/test_conditional_data_driven.py
-tests/unit/operators/test_conditional_edge_cases.py
-tests/unit/operators/test_conditional_error_handling.py
-tests/unit/operators/test_conditional_parameters.py
-tests/unit/operators/test_conditional_performance.py
+tests/unit/operators/test_compose_conditions.py
+tests/unit/operators/test_conditional_auto_detection.py
+tests/unit/operators/test_conditional_core.py
 tests/unit/operators/test_conditional_real_world.py
 tests/unit/operators/test_operators.py
 tests/unit/operators/test_primitives_complete.py

{datacompose-0.2.5.2 → datacompose-0.2.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datacompose"
-version = "0.2.5.2"
+version = "0.2.6.0"
 description = "Copy-pasteable data transformation primitives for PySpark. Inspired by shadcn-svelte."
 authors = [
     {name = "Datacompose Contributors"},