datacompose 0.2.4__py3-none-any.whl → 0.2.5.2__py3-none-any.whl

datacompose/cli/commands/add.py

@@ -2,12 +2,12 @@
 Add command for generating UDFs.
 """
 
-import json
 from pathlib import Path
 
 import click
 
 from datacompose.cli.colors import dim, error, highlight, info, success
+from datacompose.cli.config import ConfigLoader
 from datacompose.cli.validation import validate_platform, validate_type_for_platform
 from datacompose.transformers.discovery import TransformerDiscovery
 
@@ -86,28 +86,48 @@ _MODULE_DIR = Path(__file__).parent
 @click.option(
     "--target",
     "-t",
-    default="pyspark",
+    default=None,
     shell_complete=complete_target,
-    help="Target platform (e.g., 'pyspark', 'postgres', 'snowflake'). Default: pyspark",
+    help="Target platform (e.g., 'pyspark', 'postgres', 'snowflake'). Uses default from datacompose.json if not specified",
 )
 @click.option(
     "--type",
     shell_complete=complete_type,
     help="UDF type for the platform (e.g., 'pandas_udf', 'sql_udf'). Uses platform default if not specified",
 )
-@click.option("--output", "-o", help="Output directory (default: build/{target})")
 @click.option(
-    "--template-dir",
-    default="src/transformers/templates",
-    help="Directory containing templates (default: src/transformers/templates)",
+    "--output",
+    "-o",
+    help="Output directory (default: from config or transformers/{target})",
 )
 @click.option("--verbose", "-v", is_flag=True, help="Verbose output")
 @click.pass_context
-def add(ctx, transformer, target, type, output, template_dir, verbose):
+def add(ctx, transformer, target, type, output, verbose):
     """Add UDFs for transformers.
 
-    TRANSFORMER: Transformer to add UDF for (e.g., 'clean_emails')
+    TRANSFORMER: Transformer to add UDF for (e.g., 'emails')
     """
+    # Load config to get default target if not specified
+    config = ConfigLoader.load_config()
+
+    if target is None:
+        # Try to get default target from config
+        target = ConfigLoader.get_default_target(config)
+        if target is None:
+            print(
+                error(
+                    "Error: No target specified and no default target found in datacompose.json"
+                )
+            )
+            print(
+                info(
+                    "Please specify a target with --target or run 'datacompose init' to set up defaults"
+                )
+            )
+            ctx.exit(1)
+        elif verbose:
+            print(dim(f"Using default target from config: {target}"))
+
     # Initialize discovery for validation
     discovery = TransformerDiscovery()
 
@@ -120,12 +140,12 @@ def add(ctx, transformer, target, type, output, template_dir, verbose):
         ctx.exit(1)
 
     # Combine target and type into generator reference
-    exit_code = _run_add(transformer, target, output, template_dir, verbose)
+    exit_code = _run_add(transformer, target, output, verbose)
     if exit_code != 0:
         ctx.exit(exit_code)
 
 
-def _run_add(transformer, target, output, template_dir, verbose) -> int:
+def _run_add(transformer, target, output, verbose) -> int:
     """Execute the add command."""
     # Initialize discovery
     discovery = TransformerDiscovery()
@@ -136,9 +156,7 @@ def _run_add(transformer, target, output, template_dir, verbose) -> int:
     if not transformer_path:
         print(error(f"Error: Transformer not found: {transformer}"))
         print(
-            info(
-                f"Available transformers: {', '.join(discovery.list_transformers())}"
-            )
+            info(f"Available transformers: {', '.join(discovery.list_transformers())}")
         )
         return 1
     else:
@@ -156,20 +174,27 @@ def _run_add(transformer, target, output, template_dir, verbose) -> int:
         return 1
 
     # Determine output directory
-    # Extract platform from target (e.g., "pyspark.pandas_udf" -> "pyspark")
-    platform = target.split(".")[0]
-
     if not output:
-        output_dir = f"build/{platform}/{transformer_name}"
+        # Try to get output from config first
+        config = ConfigLoader.load_config()
+        config_output = ConfigLoader.get_target_output(config, target)
+        if config_output:
+            # Config output already includes 'transformers/pyspark', so use it directly
+            output_dir = config_output
+        else:
+            output_dir = f"transformers/{target}"
     else:
-        output_dir = f"{output}/{platform}/{transformer_name}"
-
-    # Create generator instance
-    generator = generator_class(
-        template_dir=Path(template_dir), output_dir=Path(output_dir), verbose=verbose
-    )
+        output_dir = output
 
     try:
+        # Create generator instance
+        # Note: template_dir is required by base class but not used by current generators
+        generator = generator_class(
+            template_dir=Path("."),  # Placeholder - not actually used
+            output_dir=Path(output_dir),
+            verbose=verbose,
+        )
+
         # Generate the UDF
         result = generator.generate(
             transformer_name, force=False, transformer_dir=transformer_dir
@@ -182,13 +207,15 @@ def _run_add(transformer, target, output, template_dir, verbose) -> int:
                 print(dim(f"   Hash: {result.get('hash', 'N/A')}"))
         else:
             print(success(f"✓ UDF generated: {result['output_path']}"))
-            print(success(f"✓ Test created: {result['test_path']}"))
+            if result.get("test_path"):
+                print(success(f"✓ Test created: {result['test_path']}"))
             print(highlight(f"Function name: {result['function_name']}"))
             if verbose:
                 print(dim(f"   Target: {target}"))
                 print(highlight("\nGenerated package contents:"))
                 print(f"  - UDF code: {result['output_path']}")
-                print(f"  - Test file: {result['test_path']}")
+                if result.get("test_path"):
+                    print(f"  - Test file: {result['test_path']}")
 
         return 0
 
@@ -199,17 +226,3 @@ def _run_add(transformer, target, output, template_dir, verbose) -> int:
 
             traceback.print_exc()
         return 1
-
-
-def _load_config() -> dict:
-    """Load datacompose.json configuration if it exists."""
-    config_path = Path("datacompose.json")
-    if config_path.exists():
-        try:
-            with open(config_path, "r") as f:
-                return json.load(f)
-        except Exception:
-            pass
-    return {}
-
-

datacompose/cli/commands/init.py

@@ -18,10 +18,11 @@ from datacompose.cli.colors import dim, error, highlight, info, success
 
 DEFAULT_CONFIG = {
     "version": "1.0",
+    "default_target": "pyspark",
     "aliases": {"utils": "./src/utils"},
     "targets": {
         "pyspark": {
-            "output": "./build/pyspark",
+            "output": "./transformers/pyspark",
         }
     },
 }
@@ -57,7 +58,7 @@ class InitCommand:
     def get_config_template(template_name: str) -> Dict[str, Any]:
         """Get configuration template by name."""
         if template_name == "minimal":
-            return {"version": "1.0", "targets": {"pyspark": {"output": "./build/pyspark"}}}
+            return {"version": "1.0", "default_target": "pyspark", "targets": {"pyspark": {"output": "./transformers/pyspark"}}}
         elif template_name == "advanced":
             config = DEFAULT_CONFIG.copy()
             config.update(
@@ -65,10 +66,10 @@ class InitCommand:
                     "style": "custom",
                     "aliases": {
                         "utils": "./src/utils",
-                        "build": "./build",
+                        "transformers": "./transformers",
                     },
                     "include": ["src/**/*"],
-                    "exclude": ["__pycache__", "build", "*.pyc", ".pytest_cache"],
+                    "exclude": ["__pycache__", "transformers", "*.pyc", ".pytest_cache"],
                     "testing": {"framework": "pytest", "test_dir": "./tests"},
                 }
             )
@@ -184,7 +185,7 @@ class InitCommand:
 
         # Select targets with multi-select
         available_targets = {
-            "pyspark": {"output": "./build/pyspark", "name": "PySpark (Apache Spark)"},
+            "pyspark": {"output": "./transformers/pyspark", "name": "PySpark (Apache Spark)"},
         }
 
         selected_targets = InitCommand.prompt_for_targets(available_targets)
@@ -199,6 +200,31 @@ class InitCommand:
 
         # Update targets with user selections
         config["targets"] = selected_targets
+        
+        # Set default target to the first selected target (or only target if single)
+        target_keys = list(selected_targets.keys())
+        if len(target_keys) == 1:
+            config["default_target"] = target_keys[0]
+        elif len(target_keys) > 1:
+            # Ask user to select default target
+            print(highlight("\nSelect Default Target"))
+            print(dim("Which platform should be used by default when running 'datacompose add'?\n"))
+            for i, key in enumerate(target_keys, 1):
+                print(f"  {i}. {key}")
+            print()
+            
+            while True:
+                choice = input(f"Select default target (1-{len(target_keys)}): ").strip()
+                try:
+                    choice_idx = int(choice) - 1
+                    if 0 <= choice_idx < len(target_keys):
+                        config["default_target"] = target_keys[choice_idx]
+                        print(dim(f"Default target set to: {target_keys[choice_idx]}\n"))
+                        break
+                    else:
+                        print(error("Invalid selection. Please try again."))
+                except ValueError:
+                    print(error("Please enter a number."))
 
         print()  # Add spacing
         return config
@@ -403,11 +429,11 @@ def _run_init(force, output, verbose, yes, skip_completion) -> int:
                     "2. Source your shell config or restart terminal for tab completion"
                 )
                 print(
-                    "3. Add your first transformer: datacompose add clean_emails --target pyspark"
+                    "3. Add your first transformer: datacompose add emails"
                 )
             else:
                 print(
-                    "2. Add your first transformer: datacompose add clean_emails --target pyspark"
+                    "2. Add your first transformer: datacompose add emails"
                 )
                 if not skip_completion:
                     print(
@@ -419,7 +445,7 @@ def _run_init(force, output, verbose, yes, skip_completion) -> int:
                 print(success("✓ Tab completion configured"))
                 print(
                     highlight(
-                        "\nRun 'datacompose add clean_emails --target pyspark' to get started"
+                        "\nRun 'datacompose add emails' to get started"
                     )
                 )
                 print(
@@ -430,7 +456,7 @@ def _run_init(force, output, verbose, yes, skip_completion) -> int:
             else:
                 print(
                     highlight(
-                        "\nRun 'datacompose add clean_emails --target pyspark' to get started"
+                        "\nRun 'datacompose add emails' to get started"
                     )
                 )
                 if not skip_completion and not yes:

datacompose/cli/commands/list.py

@@ -95,7 +95,7 @@ class ListCommand:
                 print(f"    • {transformer_name}")
 
         print("\nUsage: datacompose add <transformer> --target <platform> [--type <type>]")
-        print("Example: datacompose add clean_emails --target pyspark")
+        print("Example: datacompose add emails --target pyspark")
         return 0
 
     @staticmethod
@@ -114,5 +114,5 @@ class ListCommand:
                 print(f"    • {gen_type} ({gen_class.__name__})")
 
         print("\nUsage: datacompose add <transformer> --target <platform> [--type <type>]")
-        print("Example: datacompose add clean_emails --target pyspark")
+        print("Example: datacompose add emails --target pyspark")
         return 0

datacompose/cli/config.py

@@ -0,0 +1,80 @@
+"""
+Configuration management for Datacompose CLI.
+"""
+
+import json
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+class ConfigLoader:
+    """Load and manage Datacompose configuration."""
+    
+    DEFAULT_CONFIG_FILE = "datacompose.json"
+    
+    @staticmethod
+    def load_config(config_path: Optional[Path] = None) -> Optional[Dict[str, Any]]:
+        """Load configuration from datacompose.json.
+        
+        Args:
+            config_path: Optional path to config file. Defaults to ./datacompose.json
+            
+        Returns:
+            Config dictionary or None if not found
+        """
+        if config_path is None:
+            config_path = Path(ConfigLoader.DEFAULT_CONFIG_FILE)
+            
+        if not config_path.exists():
+            return None
+            
+        try:
+            with open(config_path, 'r') as f:
+                return json.load(f)
+        except (json.JSONDecodeError, IOError):
+            return None
+    
+    @staticmethod
+    def get_default_target(config: Optional[Dict[str, Any]] = None) -> Optional[str]:
+        """Get the default target from config.
+        
+        Args:
+            config: Optional config dict. If None, will load from file.
+            
+        Returns:
+            Default target name or None
+        """
+        if config is None:
+            config = ConfigLoader.load_config()
+            
+        if not config:
+            return None
+            
+        # Check for explicit default_target setting
+        if "default_target" in config:
+            return config["default_target"]
+            
+        # Otherwise use the first target if only one exists
+        targets = config.get("targets", {})
+        if len(targets) == 1:
+            return list(targets.keys())[0]
+            
+        return None
+    
+    @staticmethod
+    def get_target_output(config: Optional[Dict[str, Any]], target: str) -> Optional[str]:
+        """Get the output directory for a specific target.
+        
+        Args:
+            config: Config dictionary
+            target: Target name
+            
+        Returns:
+            Output directory path or None
+        """
+        if not config:
+            return None
+            
+        targets = config.get("targets", {})
+        target_config = targets.get(target, {})
+        return target_config.get("output")

datacompose/cli/main.py

@@ -25,9 +25,9 @@ def cli(ctx):
     """Generate data cleaning UDFs for various platforms.
 
     Examples:
-      datacompose init
-      datacompose add clean_emails --target pyspark
-      datacompose add clean_emails --target snowflake --output sql/udfs/
+      datacompose init                  # Set up project with default target
+      datacompose add emails            # Uses default target from config
+      datacompose add emails --target snowflake --output sql/udfs/
       datacompose list targets
     """
     pass

datacompose/generators/base.py

@@ -8,7 +8,6 @@ def __get_output_filename as well as any other build steps that you want.
 
 import hashlib
 from abc import ABC, abstractmethod
-from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, Optional
 
@@ -45,16 +44,11 @@ class BaseGenerator(ABC):
             Dictionary with generation results
         """
         # Create a minimal spec-like dict from transformer name for compatibility
-        spec = {"name": transformer_name}
+        transformer = {"name": transformer_name}
 
-        # Get template content
-        template_content = self._get_template_content(transformer_dir)
-
-        # Calculate hash for caching
-        spec_hash = self._calculate_hash(spec, template_content)
-
-        # Determine output path
-        output_file = self._get_output_filename(spec["name"])
+        file_content: str = self._get_primitives_file(transformer_dir)
+        spec_hash = self._calculate_hash(transformer, file_content)
+        output_file = self._get_output_filename(transformer["name"])
         output_path = self.output_dir / output_file
 
         # Check if regeneration is needed
@@ -63,18 +57,18 @@ class BaseGenerator(ABC):
                 "skipped": True,
                 "output_path": str(output_path),
                 "hash": spec_hash,
-                "function_name": f"{spec['name']}_udf",
+                "function_name": f"{transformer['name']}_udf",
             }
         
         # Copy utils/primitives.py to the output directory
         self._copy_utils_files(output_path)
-
+        self._write_output(output_path, file_content)
 
         return {
             "skipped": False,
             "output_path": str(output_path),
             "hash": spec_hash,
-            "function_name": f"{spec['name']}_udf",
+            "function_name": f"{transformer['name']}_udf",
         }
 
     @staticmethod
@@ -82,6 +76,7 @@ class BaseGenerator(ABC):
         """Calculate hash for cache invalidation."""
         content = str(spec) + template_content
         return hashlib.sha256(content.encode("utf-8")).hexdigest()[:8]
+    
 
     @staticmethod
     def _should_skip_generation(output_path: Path, spec_hash: str) -> bool:
@@ -100,8 +95,6 @@ class BaseGenerator(ABC):
         """Write generated content to output file."""
         # Create output directory if it doesn't exist
         output_path.parent.mkdir(parents=True, exist_ok=True)
-
-        # Create __init__.py files to make directories importable as Python packages
         self._ensure_init_files(output_path)
 
         with open(output_path, "w") as f:
@@ -112,14 +105,14 @@ class BaseGenerator(ABC):
 
     def _ensure_init_files(self, output_path: Path):
         """Ensure __init__.py files exist to make directories importable."""
-        # Get all directories from build down to the target directory
+        # Get all directories from transformers down to the target directory
         path_parts = output_path.parts
 
-        # Find the build directory index
+        # Find the transformers directory index
         try:
-            build_index = path_parts.index("build")
+            transformers_index = path_parts.index("transformers")
         except ValueError:
-            # No build directory found, just create init for immediate parent
+            # No transformers directory found, just create init for immediate parent
             init_file = output_path.parent / "__init__.py"
             if not init_file.exists():
                 init_file.touch()
@@ -127,9 +120,9 @@ class BaseGenerator(ABC):
                     print(f"Created {init_file}")
             return
 
-        # Create __init__.py files for build and all subdirectories leading to output
+        # Create __init__.py files for transformers and all subdirectories leading to output
         for i in range(
-            build_index, len(path_parts) - 1
+            transformers_index, len(path_parts) - 1
         ):  # -1 to exclude the file itself
             dir_path = Path(*path_parts[: i + 1])
             init_file = dir_path / "__init__.py"
@@ -138,25 +131,20 @@ class BaseGenerator(ABC):
                 if self.verbose:
                     print(f"Created {init_file}")
 
-    @staticmethod
-    def _prepare_template_vars(spec: Dict[str, Any], spec_hash: str) -> Dict[str, Any]:
-        """Prepare variables for template rendering."""
-        return {
-            "transformer_name": spec["name"],
-            "udf_name": f"{spec['name']}_udf",
-            "hash": spec_hash,
-            "generation_timestamp": datetime.now().isoformat(),
-            "typo_map": spec.get("typo_map", {}),
-            "regex_patterns": spec.get("regex", {}),
-            "flags": spec.get("flags", {}),
-            "options": spec.get("options", {}),
-            "custom_rules": spec.get("custom_rules", {}),
-        }
-
 
     def _copy_utils_files(self, output_path: Path):
-        """Copy utility files like primitives.py to the output directory."""
-        # Create utils directory at the same level as the output file
+        """Copy utility files like primitives.py to the transformers directory."""
+        # Find the transformers directory root
+        path_parts = output_path.parts
+        try:
+            transformers_index = path_parts.index("transformers")
+            transformers_root = Path(*path_parts[:transformers_index + 1])
+        except ValueError:
+            # Fallback to parent directory if no 'transformers' in path
+            transformers_root = output_path.parent.parent
+        
+        # Create utils directory in the same directory as the generated files
+        # This puts it at transformers/pyspark/utils
         utils_dir = output_path.parent / "utils"
         utils_dir.mkdir(parents=True, exist_ok=True)
         
@@ -179,12 +167,12 @@ class BaseGenerator(ABC):
 
     @classmethod
     @abstractmethod
-    def _get_template_location(cls, transformer_dir: Path | None) -> Path | None:
+    def _get_primitives_location(cls, transformer_dir: Path | None) -> Path | None:
         pass
 
     @abstractmethod
-    def _get_template_content(self, transformer_dir: Path | None) -> str:
-        """Get the template content for this generator."""
+    def _get_primitives_file(self, transformer_dir: Path | None) -> str:
+        """Get the file content for this generator."""
         pass
 
     @abstractmethod

datacompose/generators/pyspark/generator.py

@@ -11,41 +11,36 @@ class SparkPandasUDFGenerator(BaseGenerator):
     """Generator for Apache Spark pandas UDFs."""
 
     ENGINE_SUBDIRECTORY = "pyspark"
-    TEMPLATE_FILENAME = "pyspark_primitives.py"
+    PRIMITIVES_FILENAME = "pyspark_primitives.py"
 
     @classmethod
-    def _get_template_location(cls, transformer_dir: Path | None) -> Path | None:
+    def _get_primitives_location(cls, transformer_dir: Path | None) -> Path | None:
         if transformer_dir is None:
             return None
-        return transformer_dir / cls.ENGINE_SUBDIRECTORY / cls.TEMPLATE_FILENAME
+        return transformer_dir / cls.ENGINE_SUBDIRECTORY / cls.PRIMITIVES_FILENAME
 
-    def _get_template_content(self, transformer_dir: Path | None = None) -> str:
+    def _get_primitives_file(self, transformer_dir: Path | None = None) -> str:
         """Get the template content for Spark pandas UDFs."""
         if transformer_dir:
             # Look for transformer-specific template first
-            transformer_template = self._get_template_location(transformer_dir)
+            transformer_template = self._get_primitives_location(transformer_dir)
             if transformer_template and transformer_template.exists():
                 return transformer_template.read_text()
 
         # Fallback to generator-specific template (if it exists)
-        generator_template = Path(__file__).parent / self.TEMPLATE_FILENAME
+        generator_template = Path(__file__).parent / self.PRIMITIVES_FILENAME
         if generator_template.exists():
             return generator_template.read_text()
 
         # If no templates found, raise error
         raise FileNotFoundError(
-            f"No {self.TEMPLATE_FILENAME} template found in {transformer_dir} or {Path(__file__).parent}"
+            f"No {self.PRIMITIVES_FILENAME} template found in {transformer_dir} or {Path(__file__).parent}"
         )
 
     def _get_output_filename(self, transformer_name: str) -> str:
         """Get the output filename for PySpark primitives."""
-        # Map transformer names to their primitive namespace names
-        name_mapping = {
-            "clean_emails": "email_primitives",
-            "clean_addresses": "address_primitives", 
-            "clean_phone_numbers": "phone_primitives"
-        }
-        
-        # Use mapped name if available, otherwise fall back to transformer_name
-        output_name = name_mapping.get(transformer_name, f"{transformer_name}_primitives")
-        return f"{output_name}.py"
+        # Use the transformer name directly as the filename
+        # emails -> emails.py
+        # addresses -> addresses.py
+        # phone_numbers -> phone_numbers.py
+        return f"{transformer_name}.py"