datacompose 0.2.5.2__tar.gz → 0.2.6.1__tar.gz

{datacompose-0.2.5.2 → datacompose-0.2.6.1}/CHANGELOG.md

@@ -7,6 +7,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.6.0] - 2025-08-24
+
+### Added
+- **Automatic Conditional Detection**: Smart detection of conditional operators based on naming patterns
+  - Functions starting with `is_`, `has_`, `needs_`, `should_`, `can_`, `contains_`, `matches_`, `equals_`, `starts_with_`, `ends_with_` are automatically detected as conditionals
+  - Eliminates need for explicit `is_conditional=True` in most cases
+  - Explicit override still available when needed via `is_conditional` parameter
+- **Phone Number Processing Pipeline**: Complete phone number validation and formatting example
+  - Letter-to-number conversion (1-800-FLOWERS)
+  - NANP validation and formatting
+  - Toll-free number detection
+  - E.164 and parentheses formatting
+
+### Changed
+- **Conditional Operator Registration**: `is_conditional` parameter now optional with smart defaults
+- **Test Organization**: Consolidated conditional tests into three focused files:
+  - `test_conditional_core.py` - Core functionality, logic, errors, parameters, and performance
+  - `test_conditional_real_world.py` - Real-world pipeline scenarios
+  - `test_conditional_auto_detection.py` - Auto-detection feature tests
+
+### Fixed
+- **Phone Number Validation**: Updated NANP validation to be more flexible for testing scenarios
+
+## [0.2.5.3] - 2025-08-23
+
+### Added
+- **Compose Decorator Enhancement**: Auto-detection of PrimitiveRegistry instances in function globals
+  - Compose decorator now automatically discovers all namespace instances without explicit passing
+  - Improved namespace resolution using function's global scope instead of module globals
+  - Better support for multiple namespaces in composed functions
+
+### Fixed
+- **Namespace Resolution**: Fixed global namespace lookups to use function's own globals
+  - PipelineCompiler now correctly resolves namespaces from the decorated function's scope
+  - Fallback compose mode uses function globals for namespace discovery
+  - Prevents namespace resolution errors when registries are defined in different modules
+
+### Changed
+- **Phone Number Tests**: Updated test imports and formatting for phone number primitives
+- **Test Organization**: Added comprehensive conditional composition tests
+
 ## [0.2.5.2] - 2025-08-22
 
 ### Fixed

{datacompose-0.2.5.2/datacompose.egg-info → datacompose-0.2.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacompose
-Version: 0.2.5.2
+Version: 0.2.6.1
 Summary: Copy-pasteable data transformation primitives for PySpark. Inspired by shadcn-svelte.
 Author: Datacompose Contributors
 Maintainer: Datacompose Contributors

{datacompose-0.2.5.2 → datacompose-0.2.6.1}/datacompose/cli/__init__.py

@@ -2,4 +2,4 @@
 Datacompose CLI - Command-line interface for generating data cleaning UDFs.
 """
 
-__version__ = "0.2.4"
+__version__ = "0.2.6.0"

{datacompose-0.2.5.2 → datacompose-0.2.6.1}/datacompose/cli/commands/init.py

@@ -380,7 +380,7 @@ def _run_init(force, output, verbose, yes, skip_completion) -> int:
     # Check if config already exists
     if config_path.exists() and not force:
         print(error(f"Configuration file already exists: {config_path}"))
-        print(dim("Use --force to overwrite"))
+        print(dim("Use datacompose init --force to overwrite"))
         return 1
 
     try:

{datacompose-0.2.5.2 → datacompose-0.2.6.1}/datacompose/operators/__init__.py

@@ -18,4 +18,4 @@ __all__ = [
     "PrimitiveRegistry",
 ]
 
-__version__ = "0.2.4"
+__version__ = "0.2.6.0"

{datacompose-0.2.5.2 → datacompose-0.2.6.1}/datacompose/operators/primitives.py

@@ -16,9 +16,13 @@ from typing import Any, Callable, Dict, List, Optional, Sequence
 logger = logging.getLogger(__name__)
 
 try:
-    from pyspark.sql import Column  # type: ignore
+    from pyspark.sql import Column
+    from pyspark.sql import functions as F
 except ImportError:
-    pass
+    logging.debug("PySpark not available")
+
+# Set up module logger
+logger = logging.getLogger(__name__)
 
 
 class SmartPrimitive:
@@ -120,11 +124,15 @@ class PrimitiveRegistry:
         self._primitives = {}
         self._conditionals = {}
 
-    def register(self, name: Optional[str] = None, is_conditional: bool = False):
+    def register(
+        self, name: Optional[str] = None, is_conditional: Optional[bool] = None
+    ):
         """Decorator to register a function as a SmartPrimitive in this namespace.
 
         Args:
             name: Optional name for the primitive (defaults to function name)
+            is_conditional: Optional flag to mark as conditional. If None, auto-detects
+                          based on function name patterns.
 
         Returns:
             Decorator function that wraps the target function as a SmartPrimitive
@@ -139,7 +147,29 @@ class PrimitiveRegistry:
         def decorator(func: Callable):
             primitive_name = name or func.__name__
 
-            if is_conditional:
+            # Auto-detect conditional if not explicitly specified
+            if is_conditional is None:
+                # Check common naming patterns for conditional functions
+                conditional_patterns = [
+                    "is_",
+                    "has_",
+                    "needs_",
+                    "should_",
+                    "can_",
+                    "contains_",
+                    "matches_",
+                    "equals_",
+                    "starts_with_",
+                    "ends_with_",
+                ]
+                is_conditional_auto = any(
+                    primitive_name.startswith(pattern)
+                    for pattern in conditional_patterns
+                )
+            else:
+                is_conditional_auto = is_conditional
+
+            if is_conditional_auto:
                 self._conditionals[primitive_name] = SmartPrimitive(
                     func, primitive_name
                 )
@@ -217,9 +247,17 @@ class PrimitiveRegistry:
                 pipeline.__doc__ = func.__doc__
                 return pipeline
 
+            # Auto-detect ALL namespace instances from func.__globals__
+            # This allows using multiple namespaces without explicitly passing them
+            for var_name, var_value in func.__globals__.items():
+                if isinstance(var_value, PrimitiveRegistry):
+                    # Found a namespace instance
+                    if var_name not in namespaces:
+                        namespaces[var_name] = var_value
+
             # Try to get the function as a string and parse it
             try:
-                compiler = PipelineCompiler(namespaces, debug)
+                compiler = PipelineCompiler(namespaces, debug, func.__globals__)
                 pipeline = compiler.compile(func)
 
                 if debug and pipeline.steps:
@@ -270,7 +308,11 @@ def _fallback_compose(func: Callable, namespaces: Dict, debug: bool) -> Callable
                         method_name = node.value.func.attr
                         namespace = (
                             namespaces.get(namespace_name) if namespace_name else None
-                        ) or (globals().get(namespace_name) if namespace_name else None)
+                        ) or (
+                            func.__globals__.get(namespace_name)
+                            if namespace_name
+                            else None
+                        )
                         if namespace and hasattr(namespace, method_name):
                             method = getattr(namespace, method_name)
 
@@ -312,16 +354,6 @@ def _fallback_compose(func: Callable, namespaces: Dict, debug: bool) -> Callable
         return pipeline
 
 
-try:
-    from pyspark.sql import Column
-    from pyspark.sql import functions as F
-except ImportError:
-    logging.debug("PySpark not available")
-
-# Set up module logger
-logger = logging.getLogger(__name__)
-
-
 @dataclass
 class CompiledStep:
     """A compiled pipeline step"""
@@ -452,9 +484,15 @@ class StablePipeline:
 
 
 class PipelineCompiler:
-    def __init__(self, namespaces: Dict[str, Any], debug: bool = False):
+    def __init__(
+        self,
+        namespaces: Dict[str, Any],
+        debug: bool = False,
+        func_globals: Optional[Dict] = None,
+    ):
         self.namespaces = namespaces
         self.debug = debug
+        self.func_globals = func_globals or {}
 
     def compile(self, func: Callable) -> StablePipeline:
         try:
@@ -530,7 +568,7 @@ class PipelineCompiler:
 
             namespace = (
                 self.namespaces.get(namespace_name) if namespace_name else None
-            ) or (globals().get(namespace_name) if namespace_name else None)
+            ) or (self.func_globals.get(namespace_name) if namespace_name else None)
             if namespace and hasattr(namespace, method_name):
                 method = getattr(namespace, method_name)
 
@@ -552,7 +590,7 @@ class PipelineCompiler:
 
             namespace = (
                 self.namespaces.get(namespace_name) if namespace_name else None
-            ) or (globals().get(namespace_name) if namespace_name else None)
+            ) or (self.func_globals.get(namespace_name) if namespace_name else None)
             if namespace and hasattr(namespace, method_name):
                 method = getattr(namespace, method_name)
 

{datacompose-0.2.5.2 → datacompose-0.2.6.1}/datacompose/transformers/text/addresses/pyspark/pyspark_primitives.py

@@ -544,7 +544,7 @@ def standardize_street_prefix(
 
     Args:
         col: Column containing street prefix
-        custom_mappings: Optional dict of custom prefix mappings (case insensitive)
+        custom_mappings (Optional): Dict of custom prefix mappings (case insensitive)
 
     Returns:
         Column with standardized prefix (always abbreviated per USPS standards)
@@ -614,7 +614,7 @@ def standardize_street_suffix(
 
     Args:
         col: Column containing street suffix
-        custom_mappings: Optional dict of custom suffix mappings (case insensitive)
+        custom_mappings (Optional): Dict of custom suffix mappings (case insensitive)
 
     Returns:
         Column with standardized suffix (always abbreviated per USPS standards)
@@ -896,7 +896,7 @@ def standardize_unit_type(
 
     Args:
         col: Column containing unit type
-        custom_mappings: Optional dict of custom unit type mappings
+        custom_mappings (Optional): Dict of custom unit type mappings
 
     Returns:
         Column with standardized unit type
@@ -1206,7 +1206,7 @@ def extract_city(col: Column, custom_cities: Optional[List] = None) -> Column:
 
     Args:
         col: Column containing address text
-        custom_cities: Optional list of custom city names to recognize (case-insensitive)
+        custom_cities (Optional): List of custom city names to recognize (case-insensitive)
 
     Returns:
         Column with extracted city name or empty string if not found
@@ -1371,7 +1371,7 @@ def extract_state(col: Column, custom_states: Optional[Dict] = None) -> Column:
 
     Args:
         col: Column containing address text with state information
-        custom_states: Optional dict mapping full state names to abbreviations
+        custom_states (Optional): Dict mapping full state names to abbreviations
                       e.g., {"ONTARIO": "ON", "QUEBEC": "QC"}
 
     Returns:
@@ -1445,9 +1445,9 @@ def validate_city(
 
     Args:
         col: Column containing city names to validate
-        known_cities: Optional list of valid city names to check against
-        min_length: Minimum valid city name length (default 2)
-        max_length: Maximum valid city name length (default 50)
+        known_cities (Optional): List of valid city names to check against
+        min_length (Optional): Minimum valid city name length (default 2)
+        max_length (Optional): Maximum valid city name length (default 50)
 
     Returns:
         Boolean column indicating if city name is valid
@@ -1523,7 +1523,7 @@ def standardize_city(col: Column, custom_mappings: Optional[Dict] = None) -> Col
 
     Args:
         col: Column containing city names to standardize
-        custom_mappings: Optional dict for city name corrections/standardization
+        custom_mappings (Optional): Dict for city name corrections/standardization
                         e.g., {"ST LOUIS": "St. Louis", "NEWYORK": "New York"}
 
     Returns:
@@ -1807,7 +1807,7 @@ def standardize_country(col: Column, custom_mappings: Optional[dict] = None) ->
 
     Args:
         col: Column containing country name or abbreviation
-        custom_mappings: Optional dict of custom country mappings
+        custom_mappings (Optional): Dict of custom country mappings
 
     Returns:
         Column with standardized country name

{datacompose-0.2.5.2 → datacompose-0.2.6.1}/datacompose/transformers/text/emails/pyspark/pyspark_primitives.py

@@ -255,8 +255,8 @@ def is_valid_email(col: Column, min_length: int = 6, max_length: int = 254) -> C
 
     Args:
         col: Column containing email address
-        min_length: Minimum length for valid email
-        max_length: Maximum length for valid email
+        min_length (Optional): Minimum length for valid email (default 6)
+        max_length (Optional): Maximum length for valid email (default 254)
 
     Returns:
         Column with boolean indicating validity
@@ -286,8 +286,8 @@ def is_valid_username(col: Column, min_length: int = 1, max_length: int = 64) ->
 
     Args:
         col: Column containing email address
-        min_length: Minimum length for valid username (default 1)
-        max_length: Maximum length for valid username (default 64 per RFC)
+        min_length (Optional): Minimum length for valid username (default 1)
+        max_length (Optional): Maximum length for valid username (default 64 per RFC)
 
     Returns:
         Column with boolean indicating username validity
@@ -351,7 +351,7 @@ def is_disposable_email(
 
     Args:
         col: Column containing email address
-        disposable_domains: List of disposable domains to check against
+        disposable_domains (Optional): List of disposable domains to check against
 
     Returns:
         Column with boolean indicating if email is disposable
@@ -389,7 +389,7 @@ def is_corporate_email(
 
     Args:
         col: Column containing email address
-        free_providers: List of free email provider domains to check against
+        free_providers (Optional): List of free email provider domains to check against
 
     Returns:
         Column with boolean indicating if email is corporate
@@ -535,8 +535,8 @@ def fix_common_typos(
 
     Args:
         col: Column containing email address
-        custom_mappings: Additional domain mappings to apply (extends DOMAIN_TYPO_MAPPINGS)
-        custom_tld_mappings: Additional TLD mappings to apply (extends TLD_TYPO_MAPPINGS)
+        custom_mappings (Optional): Additional domain mappings to apply (extends DOMAIN_TYPO_MAPPINGS)
+        custom_tld_mappings (Optional): Additional TLD mappings to apply (extends TLD_TYPO_MAPPINGS)
 
     Returns:
         Column with typos fixed
@@ -604,10 +604,10 @@ def standardize_email(
 
     Args:
         col: Column containing email address
-        lowercase: Convert to lowercase
-        remove_dots_gmail: Remove dots from Gmail addresses
-        remove_plus: Remove plus addressing
-        fix_typos: Fix common domain typos
+        lowercase (Optional): Convert to lowercase (default True)
+        remove_dots_gmail (Optional): Remove dots from Gmail addresses (default True)
+        remove_plus (Optional): Remove plus addressing (default False)
+        fix_typos (Optional): Fix common domain typos (default True)
 
     Returns:
         Column with standardized email
@@ -756,8 +756,8 @@ def mask_email(col: Column, mask_char: str = "*", keep_chars: int = 3) -> Column
 
     Args:
         col: Column containing email address
-        mask_char: Character to use for masking
-        keep_chars: Number of characters to keep at start
+        mask_char (Optional): Character to use for masking (default "*")
+        keep_chars (Optional): Number of characters to keep at start (default 3)
 
     Returns:
         Column with masked email