FlowerPower 0.30.0__py3-none-any.whl → 0.31.1__py3-none-any.whl

flowerpower/cfg/__init__.py

@@ -6,6 +6,7 @@ from munch import Munch
 
 from ..settings import CONFIG_DIR, PIPELINES_DIR
 from .base import BaseConfig
+from .exceptions import ConfigLoadError, ConfigSaveError, ConfigPathError
 from .pipeline import PipelineConfig, init_pipeline_config
 from .project import ProjectConfig, init_project_config
 
@@ -21,8 +22,9 @@ class Config(BaseConfig):
         pipeline (PipelineConfig): Configuration for the pipeline.
         project (ProjectConfig): Configuration for the project.
         fs (AbstractFileSystem | None): Filesystem abstraction for I/O operations.
-        base_dir (str | Path | None): Base directory for the configuration.
-        storage_options (dict | Munch): Options for filesystem operations.
+        base_dir (str | None): Base directory for the configuration.
+        base_dir_path (pathlib.Path | None): Base directory as a Path object (property).
+        storage_options (Munch): Options for filesystem operations.
 
     Example:
         ```python
@@ -41,8 +43,61 @@ class Config(BaseConfig):
     pipeline: PipelineConfig = msgspec.field(default_factory=PipelineConfig)
     project: ProjectConfig = msgspec.field(default_factory=ProjectConfig)
     fs: AbstractFileSystem | None = None
-    base_dir: str | Path | None = None
-    storage_options: dict | Munch = msgspec.field(default_factory=Munch)
+    base_dir: str | None = None
+    storage_options: Munch = msgspec.field(default_factory=Munch)
+
+    def __post_init__(self):
+        """Handle conversion of storage_options from dict to Munch if needed."""
+        if isinstance(self.storage_options, dict):
+            self.storage_options = Munch(self.storage_options)
+        
+        # Validate storage_options
+        self._validate_storage_options()
+        
+        # Validate base_dir if provided
+        if self.base_dir is not None:
+            self._validate_base_dir()
+
+    def _validate_storage_options(self) -> None:
+        """Validate storage_options parameter.
+        
+        Raises:
+            ValueError: If storage_options contains invalid values.
+        """
+        if self.storage_options is None:
+            self.storage_options = Munch()
+        
+        if not isinstance(self.storage_options, (dict, Munch)):
+            raise ValueError(f"storage_options must be a dict or Munch, got {type(self.storage_options)}")
+
+    def _validate_base_dir(self) -> None:
+        """Validate base_dir parameter.
+        
+        Raises:
+            ValueError: If base_dir contains invalid characters or is empty.
+        """
+        # Convert Path to string if needed
+        base_dir_str = str(self.base_dir) if hasattr(self.base_dir, '__str__') else self.base_dir
+        
+        if not isinstance(base_dir_str, str):
+            raise ValueError(f"base_dir must be a string or Path, got {type(self.base_dir)}")
+        
+        # Check for directory traversal attempts (but allow absolute paths)
+        if '..' in base_dir_str:
+            raise ValueError(f"Invalid base_dir: {base_dir_str}. Contains path traversal characters.")
+        
+        # Check for empty string
+        if not base_dir_str.strip():
+            raise ValueError("base_dir cannot be empty or whitespace only.")
+
+    @property
+    def base_dir_path(self) -> Path | None:
+        """Get base_dir as a pathlib.Path object.
+        
+        Returns:
+            pathlib.Path | None: The base directory as a Path object, or None if base_dir is None.
+        """
+        return Path(self.base_dir) if self.base_dir is not None else None
 
     @classmethod
     def load(
@@ -75,21 +130,29 @@ class Config(BaseConfig):
             ```
         """
         if fs is None:
-            fs = filesystem(
-                base_dir, cached=True, dirfs=True, storage_options=storage_options
+            # Use cached filesystem for better performance
+            storage_options_hash = cls._hash_storage_options(storage_options)
+            fs = cls._get_cached_filesystem(base_dir, storage_options_hash)
+        
+        try:
+            project = ProjectConfig.load(
+                base_dir=base_dir,
+                name=name,
+                fs=fs,
+                storage_options=storage_options,
             )
-        project = ProjectConfig.load(
-            base_dir=base_dir,
-            name=name,
-            fs=fs,
-            storage_options=storage_options,
-        )
-        pipeline = PipelineConfig.load(
-            base_dir=base_dir,
-            name=pipeline_name,
-            fs=fs,
-            storage_options=storage_options,
-        )
+        except ConfigLoadError as e:
+            raise ConfigLoadError(f"Failed to load project configuration: {e}", path=base_dir, original_error=e)
+            
+        try:
+            pipeline = PipelineConfig.load(
+                base_dir=base_dir,
+                name=pipeline_name,
+                fs=fs,
+                storage_options=storage_options,
+            )
+        except ConfigLoadError as e:
+            raise ConfigLoadError(f"Failed to load pipeline configuration: {e}", path=base_dir, original_error=e)
 
         return cls(
             base_dir=base_dir,
@@ -120,9 +183,9 @@ class Config(BaseConfig):
             ```
         """
         if fs is None and self.fs is None:
-            self.fs = filesystem(
-                self.base_dir, cached=True, dirfs=True, **storage_options
-            )
+            # Use cached filesystem for better performance
+            storage_options_hash = self._hash_storage_options(storage_options)
+            self.fs = self._get_cached_filesystem(self.base_dir, storage_options_hash)
 
         if not self.fs.exists(CONFIG_DIR):
             self.fs.makedirs(CONFIG_DIR)
@@ -130,13 +193,22 @@ class Config(BaseConfig):
         if pipeline:
             self.fs.makedirs(PIPELINES_DIR, exist_ok=True)
             h_params = self.pipeline.pop("h_params") if self.pipeline.h_params else None
-            self.pipeline.to_yaml(
-                path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
-            )
+            # Validate pipeline name to prevent directory traversal
+            if self.pipeline.name and ('..' in self.pipeline.name or '/' in self.pipeline.name or '\\' in self.pipeline.name):
+                raise ConfigPathError(f"Invalid pipeline name: {self.pipeline.name}. Contains path traversal characters.", path=self.pipeline.name)
+            try:
+                self.pipeline.to_yaml(
+                    path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
+                )
+            except ConfigSaveError as e:
+                raise ConfigSaveError(f"Failed to save pipeline configuration: {e}", path=f"conf/pipelines/{self.pipeline.name}.yml", original_error=e)
             if h_params:
                 self.pipeline.h_params = h_params
         if project:
-            self.project.to_yaml("conf/project.yml", self.fs)
+            try:
+                self.project.to_yaml("conf/project.yml", self.fs)
+            except ConfigSaveError as e:
+                raise ConfigSaveError(f"Failed to save project configuration: {e}", path="conf/project.yml", original_error=e)
 
 
 def load(
@@ -247,3 +319,49 @@ def init_config(
         storage_options=storage_options,
     )
     return Config(pipeline=pipeline_cfg, project=project_cfg, fs=fs, base_dir=base_dir)
+
+
+# Helper methods for centralized load/save logic
+@classmethod
+def _load_config(
+    cls,
+    config_class: type[BaseConfig],
+    base_dir: str,
+    name: str | None,
+    fs: AbstractFileSystem,
+    storage_options: dict | BaseStorageOptions | None,
+) -> BaseConfig:
+    """Centralized configuration loading logic.
+    
+    Args:
+        config_class: The configuration class to load.
+        base_dir: Base directory for configurations.
+        name: Configuration name.
+        fs: Filesystem instance.
+        storage_options: Options for filesystem.
+        
+    Returns:
+        Loaded configuration instance.
+    """
+    return config_class.load(
+        base_dir=base_dir,
+        name=name,
+        fs=fs,
+        storage_options=storage_options,
+    )
+
+
+def _save_pipeline_config(self) -> None:
+    """Save pipeline configuration with proper handling of h_params."""
+    self.fs.makedirs(PIPELINES_DIR, exist_ok=True)
+    h_params = self.pipeline.pop("h_params") if self.pipeline.h_params else None
+    self.pipeline.to_yaml(
+        path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
+    )
+    if h_params:
+        self.pipeline.h_params = h_params
+
+
+def _save_project_config(self) -> None:
+    """Save project configuration."""
+    self.project.to_yaml("conf/project.yml", self.fs)

flowerpower/cfg/base.py

@@ -1,14 +1,104 @@
 import copy
-from typing import Any, Self
+from pathlib import Path
+from typing import Any, Self, Optional
+from functools import lru_cache
 
 import msgspec
 from fsspec_utils import AbstractFileSystem, filesystem
 from ..utils.misc import get_filesystem
+from ..utils.security import validate_file_path as security_validate_file_path
+from .exceptions import ConfigLoadError, ConfigSaveError, ConfigPathError
+
+
+def validate_file_path(path: str) -> str:
+    """
+    Validate a file path to prevent directory traversal attacks.
+    
+    Args:
+        path: The file path to validate
+        
+    Returns:
+        str: The validated path
+        
+    Raises:
+        ConfigPathError: If the path contains directory traversal attempts
+    """
+    try:
+        # Use the comprehensive security validation
+        validated_path = security_validate_file_path(
+            path,
+            allow_absolute=False,  # Config files should be relative
+            allow_relative=True
+        )
+        return str(validated_path)
+    except Exception as e:
+        # Convert security errors to config path errors for consistency
+        raise ConfigPathError(f"Invalid file path: {path}. {str(e)}", path=path) from e
 
 
 class BaseConfig(msgspec.Struct, kw_only=True):
+    # Class-level cache for filesystem instances
+    _fs_cache = {}
+    
+    @classmethod
+    @lru_cache(maxsize=32)
+    def _get_cached_filesystem(cls, base_dir: str, storage_options_hash: int) -> AbstractFileSystem:
+        """Get a cached filesystem instance.
+        
+        Args:
+            base_dir: Base directory for the filesystem.
+            storage_options_hash: Hash of storage options for cache key.
+            
+        Returns:
+            Cached filesystem instance.
+        """
+        cache_key = (base_dir, storage_options_hash)
+        if cache_key not in cls._fs_cache:
+            cls._fs_cache[cache_key] = filesystem(base_dir, cached=True, dirfs=True)
+        return cls._fs_cache[cache_key]
+    
+    @classmethod
+    def _hash_storage_options(cls, storage_options: dict | None) -> int:
+        """Create a hash of storage options for caching.
+        
+        Args:
+            storage_options: Storage options to hash.
+            
+        Returns:
+            Hash of storage options.
+        """
+        if not storage_options:
+            return hash(())
+        
+        # Convert to frozenset of items for consistent hashing
+        try:
+            return hash(frozenset(sorted(storage_options.items())))
+        except TypeError:
+            # If items are not hashable, use string representation
+            return hash(str(sorted(storage_options.items())))
     def to_dict(self) -> dict[str, Any]:
-        return msgspec.to_builtins(self)
+        # Convert to dictionary, handling special cases like type objects
+        result = {}
+        for field in self.__struct_fields__:
+            value = getattr(self, field)
+            if isinstance(value, type):
+                # Convert type objects to string representation
+                result[field] = str(value)
+            elif hasattr(value, '__struct_fields__'):
+                # Recursively convert nested msgspec structs
+                result[field] = value.to_dict()
+            elif isinstance(value, list):
+                # Handle lists that might contain type objects (like exception classes)
+                converted_list = []
+                for item in value:
+                    if isinstance(item, type):
+                        converted_list.append(str(item))
+                    else:
+                        converted_list.append(item)
+                result[field] = converted_list
+            else:
+                result[field] = value
+        return result
 
     def to_yaml(self, path: str, fs: AbstractFileSystem | None = None) -> None:
         """
@@ -19,14 +109,27 @@ class BaseConfig(msgspec.Struct, kw_only=True):
             fs: An optional filesystem instance to use for file operations.
 
         Raises:
-            NotImplementedError: If the filesystem does not support writing files.
+            ConfigSaveError: If saving the configuration fails.
+            ConfigPathError: If the path contains directory traversal attempts.
         """
-        fs = get_filesystem(fs)
+        # Validate the path to prevent directory traversal
+        try:
+            validated_path = validate_file_path(path)
+        except ConfigPathError as e:
+            raise ConfigSaveError(f"Path validation failed: {e}", path=path, original_error=e)
+            
+        # Use cached filesystem if available
+        if fs is None:
+            # Use cached filesystem if available
+            if fs is None:
+                fs = get_filesystem(fs)
         try:
-            with fs.open(path, "wb") as f:
-                f.write(msgspec.yaml.encode(self, order="deterministic"))
-        except NotImplementedError:
-            raise NotImplementedError("The filesystem does not support writing files.")
+            with fs.open(validated_path, "w") as f:
+                f.write(msgspec.yaml.encode(self, order="deterministic").decode('utf-8'))
+        except NotImplementedError as e:
+            raise ConfigSaveError("The filesystem does not support writing files.", path=validated_path, original_error=e)
+        except Exception as e:
+            raise ConfigSaveError(f"Failed to write configuration to {validated_path}", path=validated_path, original_error=e)
 
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> "BaseConfig":
@@ -52,10 +155,22 @@ class BaseConfig(msgspec.Struct, kw_only=True):
         Returns:
             An instance of the class with the values from the YAML file.
 
+        Raises:
+            ConfigLoadError: If loading the configuration fails.
+            ConfigPathError: If the path contains directory traversal attempts.
         """
+        # Validate the path to prevent directory traversal
+        try:
+            validated_path = validate_file_path(path)
+        except ConfigPathError as e:
+            raise ConfigLoadError(f"Path validation failed: {e}", path=path, original_error=e)
+            
         fs = get_filesystem(fs)
-        with fs.open(path) as f:
-            return msgspec.yaml.decode(f.read(), type=cls, strict=False)
+        try:
+            with fs.open(validated_path) as f:
+                return msgspec.yaml.decode(f.read(), type=cls, strict=True)
+        except Exception as e:
+            raise ConfigLoadError(f"Failed to load configuration from {validated_path}", path=validated_path, original_error=e)
 
     def _apply_dict_updates(self, target: Self, d: dict[str, Any]) -> None:
         """
@@ -69,8 +184,13 @@ class BaseConfig(msgspec.Struct, kw_only=True):
             if hasattr(target, k):
                 current_value = getattr(target, k)
                 if isinstance(current_value, dict) and isinstance(v, dict):
+                    # For dictionaries, update in-place to avoid deep copy
                     current_value.update(v)
+                elif hasattr(current_value, '__struct_fields__'):
+                    # For nested msgspec structs, create a new instance with merged values
+                    setattr(target, k, current_value.merge_dict(v))
                 else:
+                    # For primitive values, direct assignment is fine
                     setattr(target, k, v)
             else:
                 # Use object.__setattr__ to bypass msgspec.Struct's restrictions
@@ -96,7 +216,8 @@ class BaseConfig(msgspec.Struct, kw_only=True):
         Returns:
             A new instance of the struct with updated values.
         """
-        self_copy = copy.deepcopy(self)
+        # Use shallow copy for better performance
+        self_copy = copy.copy(self)
         self._apply_dict_updates(self_copy, d)
         return self_copy
 

flowerpower/cfg/exceptions.py

@@ -0,0 +1,53 @@
+"""
+Custom exceptions for the cfg module.
+"""
+
+from typing import Any, Dict, Optional
+
+
+class ConfigError(Exception):
+    """Base exception for configuration-related errors."""
+    pass
+
+
+class ConfigLoadError(ConfigError):
+    """Exception raised when configuration loading fails."""
+    
+    def __init__(self, message: str, path: Optional[str] = None, original_error: Optional[Exception] = None):
+        self.path = path
+        self.original_error = original_error
+        super().__init__(message)
+
+
+class ConfigSaveError(ConfigError):
+    """Exception raised when configuration saving fails."""
+    
+    def __init__(self, message: str, path: Optional[str] = None, original_error: Optional[Exception] = None):
+        self.path = path
+        self.original_error = original_error
+        super().__init__(message)
+
+
+class ConfigValidationError(ConfigError):
+    """Exception raised when configuration validation fails."""
+    
+    def __init__(self, message: str, field: Optional[str] = None, value: Any = None):
+        self.field = field
+        self.value = value
+        super().__init__(message)
+
+
+class ConfigSecurityError(ConfigError):
+    """Exception raised for security-related configuration errors."""
+    
+    def __init__(self, message: str, details: Optional[Dict[str, Any]] = None):
+        self.details = details or {}
+        super().__init__(message)
+
+
+class ConfigPathError(ConfigSecurityError):
+    """Exception raised for path-related security errors."""
+    
+    def __init__(self, message: str, path: Optional[str] = None):
+        super().__init__(message, {"path": path})
+        self.path = path