FlowerPower 0.30.0__tar.gz → 0.31.1__tar.gz

{flowerpower-0.30.0/src/FlowerPower.egg-info → flowerpower-0.31.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.30.0
+Version: 0.31.1
 Summary: A simple workflow framework for building and managing data processing pipelines
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -11,7 +11,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duration-parser>=1.0.1
 Requires-Dist: fsspec>=2024.10.0
-Requires-Dist: fsspec-utils[full]>=0.1.0
+Requires-Dist: fsspec-utils>=0.1.0
 Requires-Dist: humanize>=4.12.2
 Requires-Dist: msgspec>=0.19.0
 Requires-Dist: munch>=4.0.0

{flowerpower-0.30.0 → flowerpower-0.31.1}/pyproject.toml

@@ -4,14 +4,14 @@ description = "A simple workflow framework for building and managing data proces
 authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
 readme = "README.md"
 requires-python = ">= 3.11"
-version = "0.30.0"
+version = "0.31.1"
 keywords = ["hamilton", "workflow", "pipeline", "scheduler", "dask", "ray"]
 
 dependencies = [
   #'dill>=0.3.8',
   'duration-parser>=1.0.1',
   'fsspec>=2024.10.0',
-  'fsspec-utils[full]>=0.1.0',
+  'fsspec-utils>=0.1.0',
   'humanize>=4.12.2',
   'msgspec>=0.19.0',
   'munch>=4.0.0',
@@ -68,5 +68,75 @@ dev-dependencies = [
   "mkdocstrings>=0.30.0",
   "mkdocstrings-python>=1.17.0",
   "repomix>=0.3.4",
+  # Security audit tools
+  "bandit[toml]>=1.7.7",
+  "safety>=3.2.0",
+  "mypy>=1.13.0",
+  "pandas>=2.3.2",
+  "numpy>=2.3.3",
+  "matplotlib>=3.10.6",
+  "seaborn>=0.13.2",
 ]
 package = true
+
+# Security configuration
+[tool.bandit]
+exclude_dirs = ["tests", "examples"]
+skips = ["B101"]                     # Skip assert_used test for test files
+
+[tool.bandit.assert_used]
+skips = ["*/test_*.py", "*/tests.py"]
+
+# MyPy configuration
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+warn_unreachable = true
+strict_equality = true
+show_error_codes = true
+
+[[tool.mypy.overrides]]
+module = [
+  "hamilton.*",
+  "sf_hamilton.*",
+  "fsspec_utils.*",
+  "loguru.*",
+  "munch.*",
+  "rich.*",
+  "typer.*",
+]
+ignore_missing_imports = true
+
+# Ruff configuration (extended for security)
+[tool.ruff]
+line-length = 88
+target-version = "py311"
+
+[tool.ruff.lint]
+select = [
+  "E",  # pycodestyle errors
+  "W",  # pycodestyle warnings
+  "F",  # pyflakes
+  "I",  # isort
+  "B",  # flake8-bugbear
+  "C4", # flake8-comprehensions
+  "UP", # pyupgrade
+  "S",  # flake8-bandit (security)
+]
+ignore = [
+  "E501", # line too long, handled by black
+  "B008", # do not perform function calls in argument defaults
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S101"]    # Allow assert in tests
+"examples/*" = ["S101"] # Allow assert in examples

{flowerpower-0.30.0 → flowerpower-0.31.1/src/FlowerPower.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.30.0
+Version: 0.31.1
 Summary: A simple workflow framework for building and managing data processing pipelines
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -11,7 +11,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duration-parser>=1.0.1
 Requires-Dist: fsspec>=2024.10.0
-Requires-Dist: fsspec-utils[full]>=0.1.0
+Requires-Dist: fsspec-utils>=0.1.0
 Requires-Dist: humanize>=4.12.2
 Requires-Dist: msgspec>=0.19.0
 Requires-Dist: munch>=4.0.0

{flowerpower-0.30.0 → flowerpower-0.31.1}/src/FlowerPower.egg-info/SOURCES.txt

@@ -11,10 +11,12 @@ src/flowerpower/__init__.py
 src/flowerpower/flowerpower.py
 src/flowerpower/cfg/__init__.py
 src/flowerpower/cfg/base.py
+src/flowerpower/cfg/exceptions.py
 src/flowerpower/cfg/pipeline/__init__.py
-src/flowerpower/cfg/pipeline/_schedule.py
 src/flowerpower/cfg/pipeline/adapter.py
 src/flowerpower/cfg/pipeline/builder.py
+src/flowerpower/cfg/pipeline/builder_adapter.py
+src/flowerpower/cfg/pipeline/builder_executor.py
 src/flowerpower/cfg/pipeline/run.py
 src/flowerpower/cfg/project/__init__.py
 src/flowerpower/cfg/project/adapter.py
@@ -24,7 +26,10 @@ src/flowerpower/cli/pipeline.py
 src/flowerpower/cli/utils.py
 src/flowerpower/pipeline/__init__.py
 src/flowerpower/pipeline/base.py
+src/flowerpower/pipeline/config_manager.py
+src/flowerpower/pipeline/executor.py
 src/flowerpower/pipeline/io.py
+src/flowerpower/pipeline/lifecycle_manager.py
 src/flowerpower/pipeline/manager.py
 src/flowerpower/pipeline/pipeline.py
 src/flowerpower/pipeline/registry.py
@@ -37,10 +42,16 @@ src/flowerpower/settings/general.py
 src/flowerpower/settings/hamilton.py
 src/flowerpower/settings/logging.py
 src/flowerpower/settings/retry.py
+src/flowerpower/utils/__init__.py
+src/flowerpower/utils/adapter.py
 src/flowerpower/utils/callback.py
+src/flowerpower/utils/config.py
+src/flowerpower/utils/executor.py
+src/flowerpower/utils/filesystem.py
 src/flowerpower/utils/logging.py
 src/flowerpower/utils/misc.py
 src/flowerpower/utils/monkey.py
 src/flowerpower/utils/open_telemetry.py
+src/flowerpower/utils/security.py
 src/flowerpower/utils/templates.py
 tests/test_flowerpower_project.py

{flowerpower-0.30.0 → flowerpower-0.31.1}/src/FlowerPower.egg-info/requires.txt

@@ -1,6 +1,6 @@
 duration-parser>=1.0.1
 fsspec>=2024.10.0
-fsspec-utils[full]>=0.1.0
+fsspec-utils>=0.1.0
 humanize>=4.12.2
 msgspec>=0.19.0
 munch>=4.0.0

{flowerpower-0.30.0 → flowerpower-0.31.1}/src/flowerpower/cfg/__init__.py

@@ -6,6 +6,7 @@ from munch import Munch
 
 from ..settings import CONFIG_DIR, PIPELINES_DIR
 from .base import BaseConfig
+from .exceptions import ConfigLoadError, ConfigSaveError, ConfigPathError
 from .pipeline import PipelineConfig, init_pipeline_config
 from .project import ProjectConfig, init_project_config
 
@@ -21,8 +22,9 @@ class Config(BaseConfig):
         pipeline (PipelineConfig): Configuration for the pipeline.
         project (ProjectConfig): Configuration for the project.
         fs (AbstractFileSystem | None): Filesystem abstraction for I/O operations.
-        base_dir (str | Path | None): Base directory for the configuration.
-        storage_options (dict | Munch): Options for filesystem operations.
+        base_dir (str | None): Base directory for the configuration.
+        base_dir_path (pathlib.Path | None): Base directory as a Path object (property).
+        storage_options (Munch): Options for filesystem operations.
 
     Example:
         ```python
@@ -41,8 +43,61 @@ class Config(BaseConfig):
     pipeline: PipelineConfig = msgspec.field(default_factory=PipelineConfig)
     project: ProjectConfig = msgspec.field(default_factory=ProjectConfig)
     fs: AbstractFileSystem | None = None
-    base_dir: str | Path | None = None
-    storage_options: dict | Munch = msgspec.field(default_factory=Munch)
+    base_dir: str | None = None
+    storage_options: Munch = msgspec.field(default_factory=Munch)
+
+    def __post_init__(self):
+        """Handle conversion of storage_options from dict to Munch if needed."""
+        if isinstance(self.storage_options, dict):
+            self.storage_options = Munch(self.storage_options)
+        
+        # Validate storage_options
+        self._validate_storage_options()
+        
+        # Validate base_dir if provided
+        if self.base_dir is not None:
+            self._validate_base_dir()
+
+    def _validate_storage_options(self) -> None:
+        """Validate storage_options parameter.
+        
+        Raises:
+            ValueError: If storage_options contains invalid values.
+        """
+        if self.storage_options is None:
+            self.storage_options = Munch()
+        
+        if not isinstance(self.storage_options, (dict, Munch)):
+            raise ValueError(f"storage_options must be a dict or Munch, got {type(self.storage_options)}")
+
+    def _validate_base_dir(self) -> None:
+        """Validate base_dir parameter.
+        
+        Raises:
+            ValueError: If base_dir contains invalid characters or is empty.
+        """
+        # Convert Path to string if needed
+        base_dir_str = str(self.base_dir) if hasattr(self.base_dir, '__str__') else self.base_dir
+        
+        if not isinstance(base_dir_str, str):
+            raise ValueError(f"base_dir must be a string or Path, got {type(self.base_dir)}")
+        
+        # Check for directory traversal attempts (but allow absolute paths)
+        if '..' in base_dir_str:
+            raise ValueError(f"Invalid base_dir: {base_dir_str}. Contains path traversal characters.")
+        
+        # Check for empty string
+        if not base_dir_str.strip():
+            raise ValueError("base_dir cannot be empty or whitespace only.")
+
+    @property
+    def base_dir_path(self) -> Path | None:
+        """Get base_dir as a pathlib.Path object.
+        
+        Returns:
+            pathlib.Path | None: The base directory as a Path object, or None if base_dir is None.
+        """
+        return Path(self.base_dir) if self.base_dir is not None else None
 
     @classmethod
     def load(
@@ -75,21 +130,29 @@ class Config(BaseConfig):
             ```
         """
         if fs is None:
-            fs = filesystem(
-                base_dir, cached=True, dirfs=True, storage_options=storage_options
+            # Use cached filesystem for better performance
+            storage_options_hash = cls._hash_storage_options(storage_options)
+            fs = cls._get_cached_filesystem(base_dir, storage_options_hash)
+        
+        try:
+            project = ProjectConfig.load(
+                base_dir=base_dir,
+                name=name,
+                fs=fs,
+                storage_options=storage_options,
             )
-        project = ProjectConfig.load(
-            base_dir=base_dir,
-            name=name,
-            fs=fs,
-            storage_options=storage_options,
-        )
-        pipeline = PipelineConfig.load(
-            base_dir=base_dir,
-            name=pipeline_name,
-            fs=fs,
-            storage_options=storage_options,
-        )
+        except ConfigLoadError as e:
+            raise ConfigLoadError(f"Failed to load project configuration: {e}", path=base_dir, original_error=e)
+            
+        try:
+            pipeline = PipelineConfig.load(
+                base_dir=base_dir,
+                name=pipeline_name,
+                fs=fs,
+                storage_options=storage_options,
+            )
+        except ConfigLoadError as e:
+            raise ConfigLoadError(f"Failed to load pipeline configuration: {e}", path=base_dir, original_error=e)
 
         return cls(
             base_dir=base_dir,
@@ -120,9 +183,9 @@ class Config(BaseConfig):
             ```
         """
         if fs is None and self.fs is None:
-            self.fs = filesystem(
-                self.base_dir, cached=True, dirfs=True, **storage_options
-            )
+            # Use cached filesystem for better performance
+            storage_options_hash = self._hash_storage_options(storage_options)
+            self.fs = self._get_cached_filesystem(self.base_dir, storage_options_hash)
 
         if not self.fs.exists(CONFIG_DIR):
             self.fs.makedirs(CONFIG_DIR)
@@ -130,13 +193,22 @@ class Config(BaseConfig):
         if pipeline:
             self.fs.makedirs(PIPELINES_DIR, exist_ok=True)
             h_params = self.pipeline.pop("h_params") if self.pipeline.h_params else None
-            self.pipeline.to_yaml(
-                path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
-            )
+            # Validate pipeline name to prevent directory traversal
+            if self.pipeline.name and ('..' in self.pipeline.name or '/' in self.pipeline.name or '\\' in self.pipeline.name):
+                raise ConfigPathError(f"Invalid pipeline name: {self.pipeline.name}. Contains path traversal characters.", path=self.pipeline.name)
+            try:
+                self.pipeline.to_yaml(
+                    path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
+                )
+            except ConfigSaveError as e:
+                raise ConfigSaveError(f"Failed to save pipeline configuration: {e}", path=f"conf/pipelines/{self.pipeline.name}.yml", original_error=e)
             if h_params:
                 self.pipeline.h_params = h_params
         if project:
-            self.project.to_yaml("conf/project.yml", self.fs)
+            try:
+                self.project.to_yaml("conf/project.yml", self.fs)
+            except ConfigSaveError as e:
+                raise ConfigSaveError(f"Failed to save project configuration: {e}", path="conf/project.yml", original_error=e)
 
 
 def load(
@@ -247,3 +319,49 @@ def init_config(
         storage_options=storage_options,
     )
     return Config(pipeline=pipeline_cfg, project=project_cfg, fs=fs, base_dir=base_dir)
+
+
+# Helper methods for centralized load/save logic
+@classmethod
+def _load_config(
+    cls,
+    config_class: type[BaseConfig],
+    base_dir: str,
+    name: str | None,
+    fs: AbstractFileSystem,
+    storage_options: dict | BaseStorageOptions | None,
+) -> BaseConfig:
+    """Centralized configuration loading logic.
+    
+    Args:
+        config_class: The configuration class to load.
+        base_dir: Base directory for configurations.
+        name: Configuration name.
+        fs: Filesystem instance.
+        storage_options: Options for filesystem.
+        
+    Returns:
+        Loaded configuration instance.
+    """
+    return config_class.load(
+        base_dir=base_dir,
+        name=name,
+        fs=fs,
+        storage_options=storage_options,
+    )
+
+
+def _save_pipeline_config(self) -> None:
+    """Save pipeline configuration with proper handling of h_params."""
+    self.fs.makedirs(PIPELINES_DIR, exist_ok=True)
+    h_params = self.pipeline.pop("h_params") if self.pipeline.h_params else None
+    self.pipeline.to_yaml(
+        path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
+    )
+    if h_params:
+        self.pipeline.h_params = h_params
+
+
+def _save_project_config(self) -> None:
+    """Save project configuration."""
+    self.project.to_yaml("conf/project.yml", self.fs)

flowerpower-0.31.1/src/flowerpower/cfg/base.py

@@ -0,0 +1,262 @@
+import copy
+from pathlib import Path
+from typing import Any, Self, Optional
+from functools import lru_cache
+
+import msgspec
+from fsspec_utils import AbstractFileSystem, filesystem
+from ..utils.misc import get_filesystem
+from ..utils.security import validate_file_path as security_validate_file_path
+from .exceptions import ConfigLoadError, ConfigSaveError, ConfigPathError
+
+
+def validate_file_path(path: str) -> str:
+    """
+    Validate a file path to prevent directory traversal attacks.
+    
+    Args:
+        path: The file path to validate
+        
+    Returns:
+        str: The validated path
+        
+    Raises:
+        ConfigPathError: If the path contains directory traversal attempts
+    """
+    try:
+        # Use the comprehensive security validation
+        validated_path = security_validate_file_path(
+            path,
+            allow_absolute=False,  # Config files should be relative
+            allow_relative=True
+        )
+        return str(validated_path)
+    except Exception as e:
+        # Convert security errors to config path errors for consistency
+        raise ConfigPathError(f"Invalid file path: {path}. {str(e)}", path=path) from e
+
+
+class BaseConfig(msgspec.Struct, kw_only=True):
+    # Class-level cache for filesystem instances
+    _fs_cache = {}
+    
+    @classmethod
+    @lru_cache(maxsize=32)
+    def _get_cached_filesystem(cls, base_dir: str, storage_options_hash: int) -> AbstractFileSystem:
+        """Get a cached filesystem instance.
+        
+        Args:
+            base_dir: Base directory for the filesystem.
+            storage_options_hash: Hash of storage options for cache key.
+            
+        Returns:
+            Cached filesystem instance.
+        """
+        cache_key = (base_dir, storage_options_hash)
+        if cache_key not in cls._fs_cache:
+            cls._fs_cache[cache_key] = filesystem(base_dir, cached=True, dirfs=True)
+        return cls._fs_cache[cache_key]
+    
+    @classmethod
+    def _hash_storage_options(cls, storage_options: dict | None) -> int:
+        """Create a hash of storage options for caching.
+        
+        Args:
+            storage_options: Storage options to hash.
+            
+        Returns:
+            Hash of storage options.
+        """
+        if not storage_options:
+            return hash(())
+        
+        # Convert to frozenset of items for consistent hashing
+        try:
+            return hash(frozenset(sorted(storage_options.items())))
+        except TypeError:
+            # If items are not hashable, use string representation
+            return hash(str(sorted(storage_options.items())))
+    def to_dict(self) -> dict[str, Any]:
+        # Convert to dictionary, handling special cases like type objects
+        result = {}
+        for field in self.__struct_fields__:
+            value = getattr(self, field)
+            if isinstance(value, type):
+                # Convert type objects to string representation
+                result[field] = str(value)
+            elif hasattr(value, '__struct_fields__'):
+                # Recursively convert nested msgspec structs
+                result[field] = value.to_dict()
+            elif isinstance(value, list):
+                # Handle lists that might contain type objects (like exception classes)
+                converted_list = []
+                for item in value:
+                    if isinstance(item, type):
+                        converted_list.append(str(item))
+                    else:
+                        converted_list.append(item)
+                result[field] = converted_list
+            else:
+                result[field] = value
+        return result
+
+    def to_yaml(self, path: str, fs: AbstractFileSystem | None = None) -> None:
+        """
+        Converts the instance to a YAML file.
+
+        Args:
+            path: The path to the YAML file.
+            fs: An optional filesystem instance to use for file operations.
+
+        Raises:
+            ConfigSaveError: If saving the configuration fails.
+            ConfigPathError: If the path contains directory traversal attempts.
+        """
+        # Validate the path to prevent directory traversal
+        try:
+            validated_path = validate_file_path(path)
+        except ConfigPathError as e:
+            raise ConfigSaveError(f"Path validation failed: {e}", path=path, original_error=e)
+            
+        # Use cached filesystem if available
+        if fs is None:
+            # Use cached filesystem if available
+            if fs is None:
+                fs = get_filesystem(fs)
+        try:
+            with fs.open(validated_path, "w") as f:
+                f.write(msgspec.yaml.encode(self, order="deterministic").decode('utf-8'))
+        except NotImplementedError as e:
+            raise ConfigSaveError("The filesystem does not support writing files.", path=validated_path, original_error=e)
+        except Exception as e:
+            raise ConfigSaveError(f"Failed to write configuration to {validated_path}", path=validated_path, original_error=e)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "BaseConfig":
+        """
+        Converts a dictionary to an instance of the class.
+        Args:
+            data: The dictionary to convert.
+
+        Returns:
+            An instance of the class with the values from the dictionary.
+        """
+        return msgspec.convert(data, cls)
+
+    @classmethod
+    def from_yaml(cls, path: str, fs: AbstractFileSystem | None = None) -> "BaseConfig":
+        """
+        Loads a YAML file and converts it to an instance of the class.
+
+        Args:
+            path: The path to the YAML file.
+            fs: An optional filesystem instance to use for file operations.
+
+        Returns:
+            An instance of the class with the values from the YAML file.
+
+        Raises:
+            ConfigLoadError: If loading the configuration fails.
+            ConfigPathError: If the path contains directory traversal attempts.
+        """
+        # Validate the path to prevent directory traversal
+        try:
+            validated_path = validate_file_path(path)
+        except ConfigPathError as e:
+            raise ConfigLoadError(f"Path validation failed: {e}", path=path, original_error=e)
+            
+        fs = get_filesystem(fs)
+        try:
+            with fs.open(validated_path) as f:
+                return msgspec.yaml.decode(f.read(), type=cls, strict=True)
+        except Exception as e:
+            raise ConfigLoadError(f"Failed to load configuration from {validated_path}", path=validated_path, original_error=e)
+
+    def _apply_dict_updates(self, target: Self, d: dict[str, Any]) -> None:
+        """
+        Helper method to apply dictionary updates to a target instance.
+        
+        Args:
+            target: The target instance to apply updates to.
+            d: The dictionary containing updates to apply.
+        """
+        for k, v in d.items():
+            if hasattr(target, k):
+                current_value = getattr(target, k)
+                if isinstance(current_value, dict) and isinstance(v, dict):
+                    # For dictionaries, update in-place to avoid deep copy
+                    current_value.update(v)
+                elif hasattr(current_value, '__struct_fields__'):
+                    # For nested msgspec structs, create a new instance with merged values
+                    setattr(target, k, current_value.merge_dict(v))
+                else:
+                    # For primitive values, direct assignment is fine
+                    setattr(target, k, v)
+            else:
+                # Use object.__setattr__ to bypass msgspec.Struct's restrictions
+                object.__setattr__(target, k, v)
+
+    def update(self, d: dict[str, Any]) -> None:
+        """
+        Updates this instance with values from the provided dictionary.
+        
+        Args:
+            d: The dictionary containing updates to apply.
+        """
+        self._apply_dict_updates(self, d)
+
+    def merge_dict(self, d: dict[str, Any]) -> Self:
+        """
+        Creates a copy of this instance and updates the copy with values
+        from the provided dictionary. The original instance (self) is not modified.
+
+        Args:
+            d: The dictionary to get values from.
+
+        Returns:
+            A new instance of the struct with updated values.
+        """
+        # Use shallow copy for better performance
+        self_copy = copy.copy(self)
+        self._apply_dict_updates(self_copy, d)
+        return self_copy
+
+    def merge(self, source: Self) -> Self:
+        """
+        Creates a copy of this instance and updates the copy with values
+        from the source struct, only if the source field's value is not
+        its default value. The original instance (self) is not modified.
+
+        Args:
+            source: The msgspec.Struct instance of the same type to get values from.
+
+        Returns:
+            A new instance of the struct with updated values.
+
+        Raises:
+            TypeError: If source is not of the same type as self.
+        """
+        if type(self) is not type(source):
+            raise TypeError(
+                f"Source must be an instance of {type(self).__name__}, not {type(source).__name__}"
+            )
+
+        updated_instance = copy.copy(self)
+
+        # Get default values if they exist
+        defaults = getattr(source, "__struct_defaults__", {})
+
+        for field in source.__struct_fields__:
+            source_value = getattr(source, field)
+            has_explicit_default = field in defaults
+            is_default_value = False
+
+            if has_explicit_default:
+                is_default_value = source_value == defaults[field]
+            else:
+                is_default_value = source_value is None
+
+            if not is_default_value:
+                setattr(updated_instance, field, source_value)
+
+        return updated_instance