FlowerPower 0.21.0__tar.gz → 0.31.0__tar.gz

{flowerpower-0.21.0/src/FlowerPower.egg-info → flowerpower-0.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.21.0
+Version: 0.31.0
 Summary: A simple workflow framework for building and managing data processing pipelines
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -25,26 +25,14 @@ Provides-Extra: io
 Requires-Dist: flowerpower-io>=0.1.1; extra == "io"
 Provides-Extra: io-legacy
 Requires-Dist: flowerpower-io[legacy]>=0.1.1; extra == "io-legacy"
-Provides-Extra: mongodb
-Requires-Dist: pymongo>=4.7.2; extra == "mongodb"
-Provides-Extra: mqtt
-Requires-Dist: paho-mqtt>=2.1.0; extra == "mqtt"
-Requires-Dist: orjson>=3.10.11; extra == "mqtt"
-Requires-Dist: mmh3>=5.1.0; extra == "mqtt"
 Provides-Extra: opentelemetry
 Requires-Dist: opentelemetry-api>=1.5.0; extra == "opentelemetry"
 Requires-Dist: opentelemetry-sdk>=1.5.0; extra == "opentelemetry"
 Requires-Dist: opentelemetry-exporter-jaeger>=1.21.0; extra == "opentelemetry"
 Provides-Extra: ray
 Requires-Dist: ray>=2.34.0; extra == "ray"
-Provides-Extra: tui
-Requires-Dist: textual>=0.85.2; extra == "tui"
 Provides-Extra: ui
 Requires-Dist: sf-hamilton-ui>=0.0.11; extra == "ui"
-Provides-Extra: webserver
-Requires-Dist: sanic>=24.6.0; extra == "webserver"
-Requires-Dist: sanic-ext>=23.12.0; extra == "webserver"
-Requires-Dist: orjson>=3.10.11; extra == "webserver"
 Provides-Extra: openlineage
 Requires-Dist: openlineage-python>=1.32.0; extra == "openlineage"
 Dynamic: license-file

flowerpower-0.31.0/pyproject.toml

@@ -0,0 +1,142 @@
+[project]
+name = "FlowerPower"
+description = "A simple workflow framework for building and managing data processing pipelines"
+authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
+readme = "README.md"
+requires-python = ">= 3.11"
+version = "0.31.0"
+keywords = ["hamilton", "workflow", "pipeline", "scheduler", "dask", "ray"]
+
+dependencies = [
+  #'dill>=0.3.8',
+  'duration-parser>=1.0.1',
+  'fsspec>=2024.10.0',
+  'fsspec-utils>=0.1.0',
+  'humanize>=4.12.2',
+  'msgspec>=0.19.0',
+  'munch>=4.0.0',
+  #"openai>=1.100.2",
+  #'orjson>=3.10.15',
+  #'python-dotenv>=1.0.1',
+  'pyyaml>=6.0.1',
+  'rich>=13.9.3',
+  's3fs>=2024.10.0',
+  'sf-hamilton-sdk>=0.5.2',
+  'sf-hamilton[visualization,rich,tqdm]>=1.69.0',
+  'typer>=0.12.3',
+]
+
+
+[project.urls]
+"Homepage" = "https://github.com/legout/flowerpower"
+"Bug Tracker" = "https://github.com/legout/flowerpower/issues"
+
+[project.scripts]
+flowerpower = "flowerpower.cli:app"
+
+[project.optional-dependencies]
+io = ["flowerpower-io>=0.1.1"]
+io-legacy = ["flowerpower-io[legacy]>=0.1.1"]
+opentelemetry = [
+  "opentelemetry-api>=1.5.0",
+  "opentelemetry-sdk>=1.5.0",
+  "opentelemetry-exporter-jaeger>=1.21.0", #"sf-hamilton[opentelemetry]>=1.83.3"
+]
+ray = ["ray>=2.34.0"]
+ui = ["sf-hamilton-ui>=0.0.11"]
+
+openlineage = ["openlineage-python>=1.32.0"]
+
+
+[tool.uv]
+dev-dependencies = [
+  "ipython>=8.24.0",
+  "isort>=5.13.2",
+  "ruff>=0.7.1",
+  "jupyterlab>=4.3.0",
+  "pytest>=8.3.4",
+  "pytest-mock>=3.12.0",
+  "pytest-cov>=4.1.0",
+  "marimo>=0.10.19",
+  "pre-commit>=4.2.0",
+  "mkdocs>=1.6.1",
+  "mkdocs-material>=9.6.17",
+  "quarto>=0.1.0",
+  "mkdocs-glightbox>=0.4.0",
+  "mkdocs-mermaid2-plugin>=1.2.1",
+  "pymdown-extensions>=10.16.1",
+  "mkdocstrings>=0.30.0",
+  "mkdocstrings-python>=1.17.0",
+  "repomix>=0.3.4",
+  # Security audit tools
+  "bandit[toml]>=1.7.7",
+  "safety>=3.2.0",
+  "mypy>=1.13.0",
+  "pandas>=2.3.2",
+  "numpy>=2.3.3",
+  "matplotlib>=3.10.6",
+  "seaborn>=0.13.2",
+]
+package = true
+
+# Security configuration
+[tool.bandit]
+exclude_dirs = ["tests", "examples"]
+skips = ["B101"]                     # Skip assert_used test for test files
+
+[tool.bandit.assert_used]
+skips = ["*/test_*.py", "*/tests.py"]
+
+# MyPy configuration
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+warn_unreachable = true
+strict_equality = true
+show_error_codes = true
+
+[[tool.mypy.overrides]]
+module = [
+  "hamilton.*",
+  "sf_hamilton.*",
+  "fsspec_utils.*",
+  "loguru.*",
+  "munch.*",
+  "rich.*",
+  "typer.*",
+]
+ignore_missing_imports = true
+
+# Ruff configuration (extended for security)
+[tool.ruff]
+line-length = 88
+target-version = "py311"
+
+[tool.ruff.lint]
+select = [
+  "E",  # pycodestyle errors
+  "W",  # pycodestyle warnings
+  "F",  # pyflakes
+  "I",  # isort
+  "B",  # flake8-bugbear
+  "C4", # flake8-comprehensions
+  "UP", # pyupgrade
+  "S",  # flake8-bandit (security)
+]
+ignore = [
+  "E501", # line too long, handled by black
+  "B008", # do not perform function calls in argument defaults
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S101"]    # Allow assert in tests
+"examples/*" = ["S101"] # Allow assert in examples

{flowerpower-0.21.0 → flowerpower-0.31.0/src/FlowerPower.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.21.0
+Version: 0.31.0
 Summary: A simple workflow framework for building and managing data processing pipelines
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -25,26 +25,14 @@ Provides-Extra: io
 Requires-Dist: flowerpower-io>=0.1.1; extra == "io"
 Provides-Extra: io-legacy
 Requires-Dist: flowerpower-io[legacy]>=0.1.1; extra == "io-legacy"
-Provides-Extra: mongodb
-Requires-Dist: pymongo>=4.7.2; extra == "mongodb"
-Provides-Extra: mqtt
-Requires-Dist: paho-mqtt>=2.1.0; extra == "mqtt"
-Requires-Dist: orjson>=3.10.11; extra == "mqtt"
-Requires-Dist: mmh3>=5.1.0; extra == "mqtt"
 Provides-Extra: opentelemetry
 Requires-Dist: opentelemetry-api>=1.5.0; extra == "opentelemetry"
 Requires-Dist: opentelemetry-sdk>=1.5.0; extra == "opentelemetry"
 Requires-Dist: opentelemetry-exporter-jaeger>=1.21.0; extra == "opentelemetry"
 Provides-Extra: ray
 Requires-Dist: ray>=2.34.0; extra == "ray"
-Provides-Extra: tui
-Requires-Dist: textual>=0.85.2; extra == "tui"
 Provides-Extra: ui
 Requires-Dist: sf-hamilton-ui>=0.0.11; extra == "ui"
-Provides-Extra: webserver
-Requires-Dist: sanic>=24.6.0; extra == "webserver"
-Requires-Dist: sanic-ext>=23.12.0; extra == "webserver"
-Requires-Dist: orjson>=3.10.11; extra == "webserver"
 Provides-Extra: openlineage
 Requires-Dist: openlineage-python>=1.32.0; extra == "openlineage"
 Dynamic: license-file

{flowerpower-0.21.0 → flowerpower-0.31.0}/src/FlowerPower.egg-info/SOURCES.txt

@@ -11,27 +11,30 @@ src/flowerpower/__init__.py
 src/flowerpower/flowerpower.py
 src/flowerpower/cfg/__init__.py
 src/flowerpower/cfg/base.py
+src/flowerpower/cfg/exceptions.py
 src/flowerpower/cfg/pipeline/__init__.py
-src/flowerpower/cfg/pipeline/_schedule.py
 src/flowerpower/cfg/pipeline/adapter.py
 src/flowerpower/cfg/pipeline/builder.py
+src/flowerpower/cfg/pipeline/builder_adapter.py
+src/flowerpower/cfg/pipeline/builder_executor.py
 src/flowerpower/cfg/pipeline/run.py
 src/flowerpower/cfg/project/__init__.py
 src/flowerpower/cfg/project/adapter.py
 src/flowerpower/cli/__init__.py
 src/flowerpower/cli/cfg.py
-src/flowerpower/cli/mqtt.py
 src/flowerpower/cli/pipeline.py
 src/flowerpower/cli/utils.py
 src/flowerpower/pipeline/__init__.py
 src/flowerpower/pipeline/base.py
+src/flowerpower/pipeline/config_manager.py
+src/flowerpower/pipeline/executor.py
 src/flowerpower/pipeline/io.py
+src/flowerpower/pipeline/lifecycle_manager.py
 src/flowerpower/pipeline/manager.py
 src/flowerpower/pipeline/pipeline.py
 src/flowerpower/pipeline/registry.py
 src/flowerpower/pipeline/visualizer.py
 src/flowerpower/plugins/io/__init__.py
-src/flowerpower/plugins/mqtt/__init__.py
 src/flowerpower/settings/__init__.py
 src/flowerpower/settings/_backend.py
 src/flowerpower/settings/executor.py
@@ -39,10 +42,16 @@ src/flowerpower/settings/general.py
 src/flowerpower/settings/hamilton.py
 src/flowerpower/settings/logging.py
 src/flowerpower/settings/retry.py
+src/flowerpower/utils/__init__.py
+src/flowerpower/utils/adapter.py
 src/flowerpower/utils/callback.py
+src/flowerpower/utils/config.py
+src/flowerpower/utils/executor.py
+src/flowerpower/utils/filesystem.py
 src/flowerpower/utils/logging.py
 src/flowerpower/utils/misc.py
 src/flowerpower/utils/monkey.py
 src/flowerpower/utils/open_telemetry.py
+src/flowerpower/utils/security.py
 src/flowerpower/utils/templates.py
 tests/test_flowerpower_project.py

{flowerpower-0.21.0 → flowerpower-0.31.0}/src/FlowerPower.egg-info/requires.txt

@@ -17,14 +17,6 @@ flowerpower-io>=0.1.1
 [io-legacy]
 flowerpower-io[legacy]>=0.1.1
 
-[mongodb]
-pymongo>=4.7.2
-
-[mqtt]
-paho-mqtt>=2.1.0
-orjson>=3.10.11
-mmh3>=5.1.0
-
 [openlineage]
 openlineage-python>=1.32.0
 
@@ -36,13 +28,5 @@ opentelemetry-exporter-jaeger>=1.21.0
 [ray]
 ray>=2.34.0
 
-[tui]
-textual>=0.85.2
-
 [ui]
 sf-hamilton-ui>=0.0.11
-
-[webserver]
-sanic>=24.6.0
-sanic-ext>=23.12.0
-orjson>=3.10.11

{flowerpower-0.21.0 → flowerpower-0.31.0}/src/flowerpower/cfg/__init__.py

@@ -6,6 +6,7 @@ from munch import Munch
 
 from ..settings import CONFIG_DIR, PIPELINES_DIR
 from .base import BaseConfig
+from .exceptions import ConfigLoadError, ConfigSaveError, ConfigPathError
 from .pipeline import PipelineConfig, init_pipeline_config
 from .project import ProjectConfig, init_project_config
 
@@ -21,8 +22,9 @@ class Config(BaseConfig):
         pipeline (PipelineConfig): Configuration for the pipeline.
         project (ProjectConfig): Configuration for the project.
         fs (AbstractFileSystem | None): Filesystem abstraction for I/O operations.
-        base_dir (str | Path | None): Base directory for the configuration.
-        storage_options (dict | Munch): Options for filesystem operations.
+        base_dir (str | None): Base directory for the configuration.
+        base_dir_path (pathlib.Path | None): Base directory as a Path object (property).
+        storage_options (Munch): Options for filesystem operations.
 
     Example:
         ```python
@@ -41,8 +43,61 @@ class Config(BaseConfig):
     pipeline: PipelineConfig = msgspec.field(default_factory=PipelineConfig)
     project: ProjectConfig = msgspec.field(default_factory=ProjectConfig)
     fs: AbstractFileSystem | None = None
-    base_dir: str | Path | None = None
-    storage_options: dict | Munch = msgspec.field(default_factory=Munch)
+    base_dir: str | None = None
+    storage_options: Munch = msgspec.field(default_factory=Munch)
+
+    def __post_init__(self):
+        """Handle conversion of storage_options from dict to Munch if needed."""
+        if isinstance(self.storage_options, dict):
+            self.storage_options = Munch(self.storage_options)
+        
+        # Validate storage_options
+        self._validate_storage_options()
+        
+        # Validate base_dir if provided
+        if self.base_dir is not None:
+            self._validate_base_dir()
+
+    def _validate_storage_options(self) -> None:
+        """Validate storage_options parameter.
+        
+        Raises:
+            ValueError: If storage_options contains invalid values.
+        """
+        if self.storage_options is None:
+            self.storage_options = Munch()
+        
+        if not isinstance(self.storage_options, (dict, Munch)):
+            raise ValueError(f"storage_options must be a dict or Munch, got {type(self.storage_options)}")
+
+    def _validate_base_dir(self) -> None:
+        """Validate base_dir parameter.
+        
+        Raises:
+            ValueError: If base_dir contains invalid characters or is empty.
+        """
+        # Convert Path to string if needed
+        base_dir_str = str(self.base_dir) if hasattr(self.base_dir, '__str__') else self.base_dir
+        
+        if not isinstance(base_dir_str, str):
+            raise ValueError(f"base_dir must be a string or Path, got {type(self.base_dir)}")
+        
+        # Check for directory traversal attempts (but allow absolute paths)
+        if '..' in base_dir_str:
+            raise ValueError(f"Invalid base_dir: {base_dir_str}. Contains path traversal characters.")
+        
+        # Check for empty string
+        if not base_dir_str.strip():
+            raise ValueError("base_dir cannot be empty or whitespace only.")
+
+    @property
+    def base_dir_path(self) -> Path | None:
+        """Get base_dir as a pathlib.Path object.
+        
+        Returns:
+            pathlib.Path | None: The base directory as a Path object, or None if base_dir is None.
+        """
+        return Path(self.base_dir) if self.base_dir is not None else None
 
     @classmethod
     def load(
@@ -75,21 +130,29 @@ class Config(BaseConfig):
             ```
         """
         if fs is None:
-            fs = filesystem(
-                base_dir, cached=True, dirfs=True, storage_options=storage_options
+            # Use cached filesystem for better performance
+            storage_options_hash = cls._hash_storage_options(storage_options)
+            fs = cls._get_cached_filesystem(base_dir, storage_options_hash)
+        
+        try:
+            project = ProjectConfig.load(
+                base_dir=base_dir,
+                name=name,
+                fs=fs,
+                storage_options=storage_options,
             )
-        project = ProjectConfig.load(
-            base_dir=base_dir,
-            name=name,
-            fs=fs,
-            storage_options=storage_options,
-        )
-        pipeline = PipelineConfig.load(
-            base_dir=base_dir,
-            name=pipeline_name,
-            fs=fs,
-            storage_options=storage_options,
-        )
+        except ConfigLoadError as e:
+            raise ConfigLoadError(f"Failed to load project configuration: {e}", path=base_dir, original_error=e)
+            
+        try:
+            pipeline = PipelineConfig.load(
+                base_dir=base_dir,
+                name=pipeline_name,
+                fs=fs,
+                storage_options=storage_options,
+            )
+        except ConfigLoadError as e:
+            raise ConfigLoadError(f"Failed to load pipeline configuration: {e}", path=base_dir, original_error=e)
 
         return cls(
             base_dir=base_dir,
@@ -120,9 +183,9 @@ class Config(BaseConfig):
             ```
         """
         if fs is None and self.fs is None:
-            self.fs = filesystem(
-                self.base_dir, cached=True, dirfs=True, **storage_options
-            )
+            # Use cached filesystem for better performance
+            storage_options_hash = self._hash_storage_options(storage_options)
+            self.fs = self._get_cached_filesystem(self.base_dir, storage_options_hash)
 
         if not self.fs.exists(CONFIG_DIR):
             self.fs.makedirs(CONFIG_DIR)
@@ -130,13 +193,22 @@ class Config(BaseConfig):
         if pipeline:
             self.fs.makedirs(PIPELINES_DIR, exist_ok=True)
             h_params = self.pipeline.pop("h_params") if self.pipeline.h_params else None
-            self.pipeline.to_yaml(
-                path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
-            )
+            # Validate pipeline name to prevent directory traversal
+            if self.pipeline.name and ('..' in self.pipeline.name or '/' in self.pipeline.name or '\\' in self.pipeline.name):
+                raise ConfigPathError(f"Invalid pipeline name: {self.pipeline.name}. Contains path traversal characters.", path=self.pipeline.name)
+            try:
+                self.pipeline.to_yaml(
+                    path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
+                )
+            except ConfigSaveError as e:
+                raise ConfigSaveError(f"Failed to save pipeline configuration: {e}", path=f"conf/pipelines/{self.pipeline.name}.yml", original_error=e)
             if h_params:
                 self.pipeline.h_params = h_params
         if project:
-            self.project.to_yaml("conf/project.yml", self.fs)
+            try:
+                self.project.to_yaml("conf/project.yml", self.fs)
+            except ConfigSaveError as e:
+                raise ConfigSaveError(f"Failed to save project configuration: {e}", path="conf/project.yml", original_error=e)
 
 
 def load(
@@ -247,3 +319,49 @@ def init_config(
         storage_options=storage_options,
     )
     return Config(pipeline=pipeline_cfg, project=project_cfg, fs=fs, base_dir=base_dir)
+
+
+# Helper methods for centralized load/save logic
+@classmethod
+def _load_config(
+    cls,
+    config_class: type[BaseConfig],
+    base_dir: str,
+    name: str | None,
+    fs: AbstractFileSystem,
+    storage_options: dict | BaseStorageOptions | None,
+) -> BaseConfig:
+    """Centralized configuration loading logic.
+    
+    Args:
+        config_class: The configuration class to load.
+        base_dir: Base directory for configurations.
+        name: Configuration name.
+        fs: Filesystem instance.
+        storage_options: Options for filesystem.
+        
+    Returns:
+        Loaded configuration instance.
+    """
+    return config_class.load(
+        base_dir=base_dir,
+        name=name,
+        fs=fs,
+        storage_options=storage_options,
+    )
+
+
+def _save_pipeline_config(self) -> None:
+    """Save pipeline configuration with proper handling of h_params."""
+    self.fs.makedirs(PIPELINES_DIR, exist_ok=True)
+    h_params = self.pipeline.pop("h_params") if self.pipeline.h_params else None
+    self.pipeline.to_yaml(
+        path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
+    )
+    if h_params:
+        self.pipeline.h_params = h_params
+
+
+def _save_project_config(self) -> None:
+    """Save project configuration."""
+    self.project.to_yaml("conf/project.yml", self.fs)