pycharter 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

pycharter/etl_generator/factory.py

@@ -1,174 +0,0 @@
-"""
-Factory for auto-discovering and creating ETL orchestrators from config directories.
-
-This module provides a factory pattern for discovering ETL pipeline configurations
-and creating orchestrators automatically, eliminating the need to manually create
-orchestrator instances for each pipeline.
-"""
-
-import logging
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
-
-from pycharter.etl_generator.orchestrator import ETLOrchestrator
-
-logger = logging.getLogger(__name__)
-
-
-class PipelineFactory:
-    """
-    Factory for auto-discovering and creating ETL orchestrators.
-    
-    Scans a config root directory for subdirectories containing ETL configuration
-    files (extract.yaml, transform.yaml, load.yaml) and provides methods to create
-    orchestrators for discovered pipelines.
-    
-    Example:
-        >>> from pycharter.etl_generator.factory import PipelineFactory
-        >>> 
-        >>> # Auto-discover pipelines from configs directory
-        >>> factory = PipelineFactory(config_root="configs")
-        >>> 
-        >>> # List discovered pipelines
-        >>> pipelines = factory.get_pipeline_names()
-        >>> print(f"Found {len(pipelines)} pipelines")
-        >>> 
-        >>> # Create orchestrator for a pipeline
-        >>> orchestrator = factory.create_orchestrator(
-        ...     pipeline_name="fmp_key_metrics",
-        ...     config_context={"FMP_API_KEY": "your_key"},
-        ...     verbose=True,
-        ... )
-        >>> 
-        >>> # Run the pipeline
-        >>> result = await orchestrator.run(symbol="AAPL")
-    """
-
-    def __init__(
-        self,
-        config_root: Union[str, Path] = "configs",
-        excluded_dirs: Optional[List[str]] = None,
-        required_files: Optional[List[str]] = None,
-    ):
-        """
-        Initialize pipeline factory.
-        
-        Args:
-            config_root: Root directory containing config directories (e.g., "configs")
-            excluded_dirs: List of directory names to exclude from discovery
-                          (default: ["old", "templates", "__pycache__", ".git"])
-            required_files: List of required files for a valid config directory
-                           (default: ["extract.yaml", "transform.yaml", "load.yaml"])
-        """
-        self.config_root = Path(config_root)
-        self.excluded_dirs = set(excluded_dirs or ["old", "templates", "__pycache__", ".git"])
-        self.required_files = required_files or [
-            "extract.yaml",
-            "transform.yaml",
-            "load.yaml",
-        ]
-        self._pipelines: Dict[str, str] = {}  # pipeline_name -> contract_dir
-        self._discover_pipelines()
-
-    def _discover_pipelines(self) -> None:
-        """Discover all valid config directories from any provider."""
-        if not self.config_root.exists():
-            logger.warning(f"Config root not found: {self.config_root}")
-            return
-
-        for provider_dir in self.config_root.iterdir():
-            if not provider_dir.is_dir() or provider_dir.name in self.excluded_dirs:
-                continue
-
-            for config_dir in provider_dir.iterdir():
-                if config_dir.is_dir() and self._is_valid_config_dir(config_dir):
-                    self._pipelines[config_dir.name] = str(config_dir)
-
-        # Log summary once to avoid duplicate logs
-        if not hasattr(PipelineFactory, "_logged_discovery"):
-            logger.debug(f"Discovered {len(self._pipelines)} pipelines from {self.config_root}")
-            PipelineFactory._logged_discovery = True
-
-    def _is_valid_config_dir(self, config_dir: Path) -> bool:
-        """
-        Check if directory contains valid ETL config files.
-        
-        Args:
-            config_dir: Directory to validate
-            
-        Returns:
-            True if directory contains all required config files
-        """
-        return all((config_dir / filename).exists() for filename in self.required_files)
-
-    def get_pipeline_names(self) -> List[str]:
-        """
-        Get list of all discovered pipeline names.
-        
-        Returns:
-            Sorted list of pipeline names
-        """
-        return sorted(self._pipelines.keys())
-
-    def get_contract_dir(self, pipeline_name: str) -> Optional[str]:
-        """
-        Get contract directory path for a pipeline.
-        
-        Args:
-            pipeline_name: Name of the pipeline
-            
-        Returns:
-            Path to contract directory, or None if not found
-        """
-        return self._pipelines.get(pipeline_name)
-
-    def create_orchestrator(
-        self,
-        pipeline_name: str,
-        config_context: Optional[Dict[str, Any]] = None,
-        verbose: bool = True,
-        **orchestrator_kwargs,
-    ) -> ETLOrchestrator:
-        """
-        Create orchestrator for a pipeline.
-        
-        Args:
-            pipeline_name: Name of pipeline (e.g., "fmp_key_metrics")
-            config_context: Optional context dictionary for value injection
-                           (e.g., {"FMP_API_KEY": "key", "TARGET_DATABASE_URL": "..."})
-            verbose: Enable verbose output
-            **orchestrator_kwargs: Additional arguments passed to ETLOrchestrator
-                                  (e.g., checkpoint_dir, progress_callback, max_memory_mb)
-        
-        Returns:
-            ETLOrchestrator instance configured for the pipeline
-            
-        Raises:
-            ValueError: If pipeline not found
-        """
-        if pipeline_name not in self._pipelines:
-            available = ", ".join(self.get_pipeline_names())
-            raise ValueError(
-                f"Pipeline '{pipeline_name}' not found. "
-                f"Available pipelines: {available}"
-            )
-
-        contract_dir = self._pipelines[pipeline_name]
-        
-        return ETLOrchestrator(
-            contract_dir=contract_dir,
-            config_context=config_context,
-            verbose=verbose,
-            **orchestrator_kwargs,
-        )
-
-    def refresh(self) -> None:
-        """
-        Refresh pipeline discovery.
-        
-        Useful if configs are added at runtime or if you want to
-        re-scan the config root directory.
-        """
-        self._pipelines.clear()
-        self._discover_pipelines()
-        logger.debug(f"Refreshed pipeline discovery: {len(self._pipelines)} pipelines found")