FlowerPower 0.30.0__py3-none-any.whl → 0.31.0__py3-none-any.whl

flowerpower/cfg/pipeline/run.py

@@ -1,7 +1,9 @@
+import warnings
 import msgspec
+import importlib
 from munch import munchify
 from typing import Any, Callable
-
+from requests.exceptions import HTTPError, ConnectionError, Timeout # Example exception
 from ... import settings
 from ..base import BaseConfig
 
@@ -22,25 +24,6 @@ class ExecutorConfig(BaseConfig):
     num_cpus: int | None = msgspec.field(default=settings.EXECUTOR_NUM_CPUS)
 
 
-class RunConfig(BaseConfig):
-    inputs: dict | None = msgspec.field(default_factory=dict)
-    final_vars: list[str] | None = msgspec.field(default_factory=list)
-    config: dict | None = msgspec.field(default_factory=dict)
-    cache: dict | bool | None = msgspec.field(default=False)
-    with_adapter: WithAdapterConfig = msgspec.field(default_factory=WithAdapterConfig)
-    executor: ExecutorConfig = msgspec.field(default_factory=ExecutorConfig)
-    log_level: str | None = msgspec.field(default="INFO")
-    max_retries: int = msgspec.field(default=3)
-    retry_delay: int | float = msgspec.field(default=1)
-    jitter_factor: float | None = msgspec.field(default=0.1)
-    retry_exceptions: list[str] = msgspec.field(default_factory=lambda: ["Exception"])
-    # New fields for comprehensive configuration
-    pipeline_adapter_cfg: dict | None = msgspec.field(default=None)
-    project_adapter_cfg: dict | None = msgspec.field(default=None)
-    adapter: dict[str, Any] | None = msgspec.field(default=None)
-    reload: bool = msgspec.field(default=False)
-
-
 class CallbackSpec(msgspec.Struct):
     """Specification for a callback function with optional arguments."""
     func: Callable
@@ -69,8 +52,8 @@ class RunConfig(BaseConfig):
     on_failure: CallbackSpec | None = msgspec.field(default=None)
 
     def __post_init__(self):
-        if isinstance(self.inputs, dict):
-            self.inputs = munchify(self.inputs)
+        # if isinstance(self.inputs, dict):
+        #     self.inputs = munchify(self.inputs)
         if isinstance(self.config, dict):
             self.config = munchify(self.config)
         if isinstance(self.cache, (dict)):
@@ -89,24 +72,8 @@ class RunConfig(BaseConfig):
             # Convert adapter instances if needed
             pass
         if isinstance(self.retry_exceptions, list):
-            # Convert string exceptions to actual exception classes
-            converted_exceptions = []
-            for exc in self.retry_exceptions:
-                if isinstance(exc, str):
-                    try:
-                        exc_class = eval(exc)
-                        # Ensure it's actually an exception class
-                        if isinstance(exc_class, type) and issubclass(exc_class, BaseException):
-                            converted_exceptions.append(exc_class)
-                        else:
-                            converted_exceptions.append(Exception)
-                    except (NameError, AttributeError):
-                        converted_exceptions.append(Exception)
-                elif isinstance(exc, type) and issubclass(exc, BaseException):
-                    converted_exceptions.append(exc)
-                else:
-                    converted_exceptions.append(Exception)
-            self.retry_exceptions = converted_exceptions
+            # Convert string exceptions to actual exception classes using dynamic import
+            self.retry_exceptions = self._convert_exception_strings(self.retry_exceptions)
 
         # Handle callback conversions
         if self.on_success is not None and not isinstance(self.on_success, CallbackSpec):
@@ -121,6 +88,98 @@ class RunConfig(BaseConfig):
                     "Invalid on_success format, must be Callable or (Callable, args, kwargs)",
                     RuntimeWarning
                 )
+    
+    def _convert_exception_strings(self, exception_list: list) -> list:
+        """Convert exception strings to actual exception classes using dynamic import.
+        
+        Args:
+            exception_list: List of exception names or classes.
+            
+        Returns:
+            List of exception classes.
+        """
+        converted_exceptions = []
+        
+        for exc in exception_list:
+            if isinstance(exc, str):
+                try:
+                    # Try to dynamically import the exception class
+                    exc_class = self._import_exception_class(exc)
+                    converted_exceptions.append(exc_class)
+                except (ImportError, AttributeError) as e:
+                    warnings.warn(
+                        f"Could not import exception class '{exc}': {e}. Using Exception instead.",
+                        RuntimeWarning
+                    )
+                    converted_exceptions.append(Exception)
+            elif isinstance(exc, type) and issubclass(exc, BaseException):
+                converted_exceptions.append(exc)
+            else:
+                warnings.warn(
+                    f"Invalid exception type: {type(exc)}. Using Exception instead.",
+                    RuntimeWarning
+                )
+                converted_exceptions.append(Exception)
+        
+        return converted_exceptions
+    
+    def _import_exception_class(self, exception_name: str) -> type:
+        """Dynamically import an exception class by name.
+        
+        Args:
+            exception_name: Name of the exception class to import.
+            
+        Returns:
+            The imported exception class.
+            
+        Raises:
+            ImportError: If the module cannot be imported.
+            AttributeError: If the exception class is not found in the module.
+        """
+        # Handle built-in exceptions first
+        built_in_exceptions = {
+            'Exception': Exception,
+            'ValueError': ValueError,
+            'TypeError': TypeError,
+            'RuntimeError': RuntimeError,
+            'FileNotFoundError': FileNotFoundError,
+            'PermissionError': PermissionError,
+            'KeyError': KeyError,
+            'AttributeError': AttributeError,
+            'ImportError': ImportError,
+            'TimeoutError': TimeoutError,
+        }
+        
+        if exception_name in built_in_exceptions:
+            return built_in_exceptions[exception_name]
+        
+        # Handle module-qualified exceptions (e.g., 'requests.exceptions.HTTPError')
+        if '.' in exception_name:
+            module_name, class_name = exception_name.rsplit('.', 1)
+            module = importlib.import_module(module_name)
+            return getattr(module, class_name)
+        
+        # Try to import from common modules
+        common_modules = [
+            'requests.exceptions',
+            'urllib.error',
+            'urllib3.exceptions',
+            'http.client',
+            'socket',
+            'os',
+            'io',
+        ]
+        
+        for module_name in common_modules:
+            try:
+                module = importlib.import_module(module_name)
+                if hasattr(module, exception_name):
+                    return getattr(module, exception_name)
+            except ImportError:
+                continue
+        
+        # If not found in common modules, raise an error
+        raise ImportError(f"Could not find exception class: {exception_name}")
 
         if self.on_failure is not None and not isinstance(self.on_failure, CallbackSpec):
             if callable(self.on_failure):

flowerpower/cfg/project/__init__.py

@@ -1,9 +1,11 @@
 import msgspec
 from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
 import posixpath
+from typing import Optional
 
 from ...settings import CONFIG_DIR
 from ..base import BaseConfig
+from ..exceptions import ConfigLoadError, ConfigSaveError, ConfigPathError
 from .adapter import AdapterConfig
 
 
@@ -36,6 +38,52 @@ class ProjectConfig(BaseConfig):
     def __post_init__(self):
         if isinstance(self.adapter, dict):
             self.adapter = AdapterConfig.from_dict(self.adapter)
+        
+        # Validate project name if provided
+        if self.name is not None:
+            self._validate_project_name()
+
+    def _validate_project_name(self) -> None:
+        """Validate project name parameter.
+        
+        Raises:
+            ValueError: If project name contains invalid characters.
+        """
+        if not isinstance(self.name, str):
+            raise ValueError(f"Project name must be a string, got {type(self.name)}")
+        
+        # Check for directory traversal attempts
+        if '..' in self.name or '/' in self.name or '\\' in self.name:
+            raise ValueError(f"Invalid project name: {self.name}. Contains path traversal characters.")
+        
+        # Check for empty string
+        if not self.name.strip():
+            raise ValueError("Project name cannot be empty or whitespace only.")
+
+    @classmethod
+    def _load_project_config(cls, fs: AbstractFileSystem, name: str | None) -> "ProjectConfig":
+        """Centralized project configuration loading logic.
+        
+        Args:
+            fs: Filesystem instance.
+            name: Project name.
+            
+        Returns:
+            Loaded project configuration.
+        """
+        if fs.exists("conf/project.yml"):
+            project = cls.from_yaml(path="conf/project.yml", fs=fs)
+        else:
+            project = cls(name=name)
+        return project
+
+    def _save_project_config(self, fs: AbstractFileSystem) -> None:
+        """Centralized project configuration saving logic.
+        
+        Args:
+            fs: Filesystem instance.
+        """
+        self.to_yaml(path=posixpath.join(CONFIG_DIR, "project.yml"), fs=fs)
 
     @classmethod
     def load(
@@ -65,15 +113,11 @@ class ProjectConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = filesystem(
-                base_dir, cached=False, dirfs=True, storage_options=storage_options
-            )
-        if fs.exists("conf/project.yml"):
-            project = ProjectConfig.from_yaml(path="conf/project.yml", fs=fs)
-        else:
-            project = ProjectConfig(name=name)
-
-        return project
+            # Use cached filesystem for better performance
+            storage_options_hash = cls._hash_storage_options(storage_options)
+            fs = cls._get_cached_filesystem(base_dir, storage_options_hash)
+        
+        return cls._load_project_config(fs, name)
 
     def save(
         self,
@@ -94,12 +138,12 @@ class ProjectConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = filesystem(
-                base_dir, cached=True, dirfs=True, storage_options=storage_options
-            )
+            # Use cached filesystem for better performance
+            storage_options_hash = self._hash_storage_options(storage_options)
+            fs = self._get_cached_filesystem(base_dir, storage_options_hash)
 
         fs.makedirs(CONFIG_DIR, exist_ok=True)
-        self.to_yaml(path=posixpath.join(CONFIG_DIR, "project.yml"), fs=fs)
+        self._save_project_config(fs)
 
 
 def init_project_config(
@@ -136,4 +180,5 @@ def init_project_config(
         storage_options=storage_options,
     )
     project.save(base_dir=base_dir, fs=fs, storage_options=storage_options)
-    return project
+    return project
+

flowerpower/cfg/project/adapter.py

@@ -1,4 +1,5 @@
 import msgspec
+import os
 from munch import munchify
 
 from ... import settings
@@ -11,6 +12,11 @@ class HamiltonTrackerConfig(BaseConfig):
     ui_url: str = msgspec.field(default=settings.HAMILTON_UI_URL)
     api_key: str | None = msgspec.field(default=None)
     verify: bool = msgspec.field(default=False)
+    
+    def __post_init__(self):
+        # Load API key from environment variable if not explicitly set
+        if self.api_key is None:
+            self.api_key = os.getenv("HAMILTON_API_KEY")
 
 
 class MLFlowConfig(BaseConfig):

flowerpower/cli/__init__.py

@@ -65,7 +65,7 @@ def init(
             parsed_storage_options = (
                 parse_dict_or_list_param(storage_options, "dict") or {}
             )
-        except Exception as e:
+        except (ValueError, SyntaxError, json.JSONDecodeError) as e:
             logger.error(f"Error parsing storage options: {e}")
             raise typer.Exit(code=1)
 
@@ -75,8 +75,14 @@ def init(
             base_dir=base_dir,
             storage_options=parsed_storage_options,
         )
+    except (FileNotFoundError, PermissionError, OSError) as e:
+        logger.error(f"File system error initializing project: {e}")
+        raise typer.Exit(code=1)
+    except ValueError as e:
+        logger.error(f"Invalid configuration for project: {e}")
+        raise typer.Exit(code=1)
     except Exception as e:
-        logger.error(f"Error initializing project: {e}")
+        logger.error(f"Unexpected error initializing project: {e}")
         raise typer.Exit(code=1)
 
 

flowerpower/cli/cfg.py

@@ -1,41 +1,3 @@
 import typer
 
 app = typer.Typer(help="Config management commands")
-
-
-# @app.command()
-# def get_project(request) -> json:
-#     cfg = request.app.ctx.pipeline_manager.cfg.project.to_dict()
-#     # cfg.pop("fs")
-#     return json({"cfg": cfg})
-
-
-# @bp.get("/pipeline/<pipeline_name>")
-# async def get_pipeline(request, pipeline_name) -> json:
-#     if pipeline_name != request.app.ctx.pipeline_manager.cfg.pipeline.name:
-#         request.app.ctx.pipeline_manager.load_config(pipeline_name)
-#     cfg = request.app.ctx.pipeline_manager.cfg.pipeline.to_dict()
-#     return json({"cfg": cfg})
-
-
-# @bp.post("/pipeline/<pipeline_name>")
-# @openapi.body({"application/json": PipelineConfig}, required=True)
-# @validate(json=PipelineConfig)
-# async def update_pipeline(request, pipeline_name, body: PipelineConfig) -> json:
-#     data = request.json
-#     if pipeline_name != request.app.ctx.pipeline_manager.cfg.pipeline.name:
-#         request.app.ctx.pipeline_manager.load_config(pipeline_name)
-#     cfg = request.app.ctx.pipeline_manager.cfg.pipeline.copy()
-#     cfg.update(data)
-#     try:
-#         cfg.to_yaml(
-#             posixpath.join(
-#                 "pipelines",
-#                 pipeline_name + ".yml",
-#             ),
-#             fs=request.app.ctx.pipeline_manager.cfg.fs,
-#         )
-#     except NotImplementedError as e:
-#         raise SanicException(f"Update failed. {e}", status_code=404)
-#     cfg
-#     return json({"cfg": cfg})