FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl

flowerpower/pipeline/registry.py

@@ -4,9 +4,12 @@
 import datetime as dt
 import os
 import posixpath
-from typing import TYPE_CHECKING
+import sys
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Dict
 
 import rich
+from fsspec_utils import AbstractFileSystem, filesystem
 from loguru import logger
 from rich.console import Console
 from rich.panel import Panel
@@ -14,18 +17,19 @@ from rich.syntax import Syntax
 from rich.table import Table
 from rich.tree import Tree
 
-from .. import settings
+from ..settings import CONFIG_DIR, LOG_LEVEL, PIPELINES_DIR
 # Import necessary config types and utility functions
 from ..cfg import PipelineConfig, ProjectConfig
-from ..fs import AbstractFileSystem
 from ..utils.logging import setup_logging
 # Assuming view_img might be used indirectly or needed later
 from ..utils.templates import (HOOK_TEMPLATE__MQTT_BUILD_CONFIG,
                                PIPELINE_PY_TEMPLATE)
+# Import base utilities
+from .base import load_module
 
 if TYPE_CHECKING:
-    # Keep this for type hinting if needed elsewhere, though Config is imported directly now
-    pass
+    from .pipeline import Pipeline
+    from ..flowerpower import FlowerPowerProject
 
 from enum import Enum
 
@@ -44,7 +48,15 @@ class HookType(str, Enum):
         return self.value
 
 
-setup_logging(level=settings.LOG_LEVEL)
+@dataclass
+class CachedPipelineData:
+    """Container for cached pipeline data."""
+    pipeline: "Pipeline"
+    config: PipelineConfig
+    module: Any
+
+
+setup_logging(level=LOG_LEVEL)
 
 
 class PipelineRegistry:
@@ -54,8 +66,8 @@ class PipelineRegistry:
         self,
         project_cfg: ProjectConfig,
         fs: AbstractFileSystem,
-        cfg_dir: str,
-        pipelines_dir: str,
+        base_dir: str | None = None,
+        storage_options: dict | None = None,
     ):
         """
         Initializes the PipelineRegistry.
@@ -63,15 +75,258 @@ class PipelineRegistry:
         Args:
             project_cfg: The project configuration object.
             fs: The filesystem instance.
-            cfg_dir: The configuration directory path.
-            pipelines_dir: The pipelines directory path.
+            base_dir: The base directory path.
+            storage_options: Storage options for filesystem operations.
         """
         self.project_cfg = project_cfg
         self._fs = fs
-        self._cfg_dir = cfg_dir
-        self._pipelines_dir = pipelines_dir
+        self._cfg_dir = CONFIG_DIR
+        self._pipelines_dir = PIPELINES_DIR
+        self._base_dir = base_dir
+        self._storage_options = storage_options or {}
         self._console = Console()
 
+        # Consolidated cache for pipeline data
+        self._pipeline_data_cache: Dict[str, CachedPipelineData] = {}
+
+        # Ensure module paths are added
+        self._add_modules_path()
+
+    @classmethod
+    def from_filesystem(
+        cls,
+        base_dir: str,
+        fs: AbstractFileSystem | None = None,
+        storage_options: dict | None = None,
+    ) -> "PipelineRegistry":
+        """
+        Create a PipelineRegistry from filesystem parameters.
+
+        This factory method creates a complete PipelineRegistry instance by:
+        1. Creating the filesystem if not provided
+        2. Loading the ProjectConfig from the base directory
+        3. Initializing the registry with the loaded configuration
+
+        Args:
+            base_dir: The base directory path for the FlowerPower project
+            fs: Optional filesystem instance. If None, will be created from base_dir
+            storage_options: Optional storage options for filesystem access
+
+        Returns:
+            PipelineRegistry: A fully configured registry instance
+
+        Raises:
+            ValueError: If base_dir is invalid or ProjectConfig cannot be loaded
+            RuntimeError: If filesystem creation fails
+
+        Example:
+            ```python
+            # Create registry from local directory
+            registry = PipelineRegistry.from_filesystem("/path/to/project")
+
+            # Create registry with S3 storage
+            registry = PipelineRegistry.from_filesystem(
+                "s3://my-bucket/project",
+                storage_options={"key": "secret"}
+            )
+            ```
+        """
+        # Create filesystem if not provided
+        if fs is None:
+            fs = filesystem(
+                base_dir,
+                storage_options=storage_options,
+                cached=storage_options is not None,
+            )
+
+        # Load project configuration
+        project_cfg = ProjectConfig.load(base_dir=base_dir, fs=fs)
+
+        # Ensure we have a ProjectConfig instance
+        if not isinstance(project_cfg, ProjectConfig):
+            raise TypeError(f"Expected ProjectConfig, got {type(project_cfg)}")
+
+        # Create and return registry instance
+        return cls(
+            project_cfg=project_cfg,
+            fs=fs,
+            base_dir=base_dir,
+            storage_options=storage_options,
+        )
+
+    def _add_modules_path(self) -> None:
+        """Add pipeline module paths to Python path."""
+        try:
+            if hasattr(self._fs, "is_cache_fs") and self._fs.is_cache_fs:
+                self._fs.sync_cache()
+                project_path = self._fs._mapper.directory
+                modules_path = posixpath.join(project_path, self._pipelines_dir)
+            else:
+                # Use the base directory directly if not using cache
+                if hasattr(self._fs, "path"):
+                    project_path = self._fs.path
+                elif self._base_dir:
+                    project_path = self._base_dir
+                else:
+                    # Fallback for mocked filesystems
+                    project_path = "."
+                modules_path = posixpath.join(project_path, self._pipelines_dir)
+
+            if project_path not in sys.path:
+                sys.path.insert(0, project_path)
+
+            if modules_path not in sys.path:
+                sys.path.insert(0, modules_path)
+        except (AttributeError, TypeError):
+            # Handle case where filesystem is mocked or doesn't have required properties
+            logger.debug("Could not add modules path - using default Python path")
+
+    # --- Pipeline Factory Methods ---
+
+    def get_pipeline(
+        self, name: str, project_context: "FlowerPowerProject", reload: bool = False
+    ) -> "Pipeline":
+        """Get a Pipeline instance for the given name.
+
+        This method creates a fully-formed Pipeline object by loading its configuration
+        and Python module, then injecting the project context.
+
+        Args:
+            name: Name of the pipeline to get
+            project_context: Reference to the FlowerPowerProject
+            reload: Whether to reload configuration and module from disk
+
+        Returns:
+            Pipeline instance ready for execution
+
+        Raises:
+            FileNotFoundError: If pipeline configuration or module doesn't exist
+            ImportError: If pipeline module cannot be imported
+            ValueError: If pipeline configuration is invalid
+        """
+        # Use cache if available and not reloading
+        if not reload and name in self._pipeline_data_cache:
+            logger.debug(f"Returning cached pipeline '{name}'")
+            return self._pipeline_data_cache[name].pipeline
+
+        logger.debug(f"Creating pipeline instance for '{name}'")
+
+        # Load pipeline configuration
+        config = self.load_config(name, reload=reload)
+
+        # Load pipeline module
+        module = self.load_module(name, reload=reload)
+
+        # Import Pipeline class here to avoid circular import
+        from .pipeline import Pipeline
+
+        # Create Pipeline instance
+        pipeline = Pipeline(
+            name=name,
+            config=config,
+            module=module,
+            project_context=project_context,
+        )
+
+        # Cache the pipeline data
+        self._pipeline_data_cache[name] = CachedPipelineData(
+            pipeline=pipeline,
+            config=config,
+            module=module,
+        )
+
+        logger.debug(f"Successfully created pipeline instance for '{name}'")
+        return pipeline
+
+    def load_config(self, name: str, reload: bool = False) -> PipelineConfig:
+        """Load pipeline configuration from disk.
+
+        Args:
+            name: Name of the pipeline
+            reload: Whether to reload from disk even if cached
+
+        Returns:
+            PipelineConfig instance
+        """
+        # Use cache if available and not reloading
+        if not reload and name in self._pipeline_data_cache:
+            logger.debug(f"Returning cached config for pipeline '{name}'")
+            return self._pipeline_data_cache[name].config
+
+        logger.debug(f"Loading configuration for pipeline '{name}'")
+
+        # Load configuration from disk
+        config = PipelineConfig.load(
+            base_dir=self._base_dir,
+            name=name,
+            fs=self._fs,
+            storage_options=self._storage_options,
+        )
+
+        # Cache the configuration (will be stored in consolidated cache when pipeline is created)
+        # For now, we'll create a temporary cache entry if it doesn't exist
+        if name not in self._pipeline_data_cache:
+            self._pipeline_data_cache[name] = CachedPipelineData(
+                pipeline=None,  # type: ignore
+                config=config,
+                module=None,  # type: ignore
+            )
+
+        return config
+
+    def load_module(self, name: str, reload: bool = False) -> Any:
+        """Load pipeline module from disk.
+
+        Args:
+            name: Name of the pipeline
+            reload: Whether to reload from disk even if cached
+
+        Returns:
+            Loaded Python module
+        """
+        # Use cache if available and not reloading
+        if not reload and name in self._pipeline_data_cache:
+            cached_data = self._pipeline_data_cache[name]
+            if cached_data.module is not None:
+                logger.debug(f"Returning cached module for pipeline '{name}'")
+                return cached_data.module
+
+        logger.debug(f"Loading module for pipeline '{name}'")
+
+        # Convert pipeline name to module name
+        formatted_name = name.replace(".", "/").replace("-", "_")
+        module_name = f"pipelines.{formatted_name}"
+
+        # Load the module
+        module = load_module(module_name, reload=reload)
+
+        # Cache the module (will be stored in consolidated cache when pipeline is created)
+        # For now, we'll update the existing cache entry if it exists
+        if name in self._pipeline_data_cache:
+            self._pipeline_data_cache[name].module = module
+        else:
+            self._pipeline_data_cache[name] = CachedPipelineData(
+                pipeline=None,  # type: ignore
+                config=None,  # type: ignore
+                module=module,
+            )
+
+        return module
+
+    def clear_cache(self, name: str | None = None):
+        """Clear cached pipelines, configurations, and modules.
+
+        Args:
+            name: If provided, clear cache only for this pipeline.
+                 If None, clear entire cache.
+        """
+        if name:
+            logger.debug(f"Clearing cache for pipeline '{name}'")
+            self._pipeline_data_cache.pop(name, None)
+        else:
+            logger.debug("Clearing entire pipeline cache")
+            self._pipeline_data_cache.clear()
+
     # --- Methods moved from PipelineManager ---
     def new(self, name: str, overwrite: bool = False):
         """
@@ -80,7 +335,6 @@ class PipelineRegistry:
         Args:
             name (str): The name of the pipeline.
             overwrite (bool): Whether to overwrite an existing pipeline. Defaults to False.
-            job_queue_type (str | None): The type of worker to use. Defaults to None.
 
         Raises:
             ValueError: If the configuration or pipeline path does not exist, or if the pipeline already exists.
@@ -101,7 +355,7 @@ class PipelineRegistry:
 
         formatted_name = name.replace(".", "/").replace("-", "_")
         pipeline_file = posixpath.join(self._pipelines_dir, f"{formatted_name}.py")
-        cfg_file = posixpath.join(self._cfg_dir, "pipelines", f"{formatted_name}.yml")
+        cfg_file = posixpath.join(self._cfg_dir, PIPELINES_DIR, f"{formatted_name}.yml")
 
         def check_and_handle(path: str):
             if self._fs.exists(path):
@@ -158,7 +412,7 @@ class PipelineRegistry:
         deleted_files = []
         if cfg:
             pipeline_cfg_path = posixpath.join(
-                self._cfg_dir, "pipelines", f"{name}.yml"
+                self._cfg_dir, PIPELINES_DIR, f"{name}.yml"
             )
             if self._fs.exists(pipeline_cfg_path):
                 self._fs.rm(pipeline_cfg_path)
@@ -204,11 +458,16 @@ class PipelineRegistry:
         """
         try:
             return self._fs.glob(posixpath.join(self._pipelines_dir, "*.py"))
-        except Exception as e:
+        except (OSError, PermissionError) as e:
             logger.error(
                 f"Error accessing pipeline directory {self._pipelines_dir}: {e}"
             )
             return []
+        except Exception as e:
+            logger.error(
+                f"Unexpected error accessing pipeline directory {self._pipelines_dir}: {e}"
+            )
+            return []
 
     def _get_names(self) -> list[str]:
         """
@@ -272,11 +531,16 @@ class PipelineRegistry:
                 except FileNotFoundError:
                     logger.warning(f"Module file not found for pipeline '{name}'")
                     pipeline_summary["module"] = "# Module file not found"
-                except Exception as e:
+                except (OSError, PermissionError, UnicodeDecodeError) as e:
                     logger.error(
                         f"Error reading module file for pipeline '{name}': {e}"
                     )
                     pipeline_summary["module"] = f"# Error reading module file: {e}"
+                except Exception as e:
+                    logger.error(
+                        f"Unexpected error reading module file for pipeline '{name}': {e}"
+                    )
+                    pipeline_summary["module"] = f"# Unexpected error reading module file: {e}"
 
             if pipeline_summary:  # Only add if cfg or code was requested and found
                 summary["pipelines"][name] = pipeline_summary
@@ -445,9 +709,12 @@ class PipelineRegistry:
                 size = f"{size_bytes / 1024:.1f} KB" if size_bytes else "0.0 KB"
             except NotImplementedError:
                 size = "N/A"
-            except Exception as e:
+            except (OSError, PermissionError) as e:
                 logger.warning(f"Could not get size for {path}: {e}")
                 size = "Error"
+            except Exception as e:
+                logger.warning(f"Unexpected error getting size for {path}: {e}")
+                size = "Error"
 
             pipeline_info.append({
                 "name": name,

flowerpower/pipeline/visualizer.py

@@ -1,12 +1,11 @@
 import posixpath
-from typing import Any
 
+from fsspec_utils import AbstractFileSystem
 from hamilton import driver
 from rich import print
 
 # Import necessary config types and utility functions
 from ..cfg import PipelineConfig, ProjectConfig
-from ..fs import AbstractFileSystem
 from ..utils.misc import view_img
 from .base import load_module  # Import module loading utility
 
@@ -26,8 +25,8 @@ class PipelineVisualizer:
         self._fs = fs
         # Attributes like fs and base_dir are accessed via self.project_cfg
 
-    def _display_all_function(self, name: str, reload: bool = False):
-        """Internal helper to load module/config and get the Hamilton DAG object.
+    def _get_dag_object(self, name: str, reload: bool = False):
+        """Get the Hamilton DAG object for a pipeline.
 
         Args:
             name (str): The name of the pipeline.
@@ -85,7 +84,7 @@ class PipelineVisualizer:
             >>> visualizer = PipelineVisualizer(project_cfg, fs)
             >>> visualizer.save_dag(name="example_pipeline", format="png")
         """
-        dag = self._display_all_function(name=name, reload=reload)
+        dag = self._get_dag_object(name=name, reload=reload)
 
         # Use project_cfg attributes for path and filesystem access
         graph_dir = posixpath.join(self.project_cfg.base_dir, "graphs")
@@ -134,7 +133,7 @@ class PipelineVisualizer:
             >>> visualizer = PipelineVisualizer(project_cfg, fs)
             >>> visualizer.show_dag(name="example_pipeline", format="png")
         """
-        dag = self._display_all_function(name=name, reload=reload)
+        dag = self._get_dag_object(name=name, reload=reload)
         if raw:
             return dag
         # Use view_img utility to display the rendered graph

flowerpower/plugins/io/__init__.py

@@ -0,0 +1,8 @@
+import warnings
+
+warnings.warn(
+    "The flowerpower.plugins.io module is deprecated. "
+    "Please use 'flowerpower-io' instead. Install it with 'pip install flowerpower-io'.",
+    DeprecationWarning,
+    stacklevel=2,
+)

flowerpower/plugins/mqtt/__init__.py

@@ -1,12 +1,8 @@
-from .cfg import MqttConfig
-from .manager import MqttManager, run_pipeline_on_message, start_listener
+import warnings
 
-MQTTManager = MqttManager
-
-__all__ = [
-    "MqttConfig",
-    "MqttManager",
-    "MQTTManager",
-    "start_listener",
-    "run_pipeline_on_message",
-]
+warnings.warn(
+    "The flowerpower.plugins.mqtt module is deprecated. "
+    "Please use 'flowerpower-mqtt' instead. Install it with 'pip install flowerpower-mqtt'.",
+    DeprecationWarning,
+    stacklevel=2,
+)

flowerpower/settings/__init__.py

@@ -1,8 +1,6 @@
 # flake8: noqa
-from .backend import *
 from .executor import *
 from .general import *
 from .hamilton import *
-from .job_queue import *
 from .logging import *
 from .retry import *

flowerpower/settings/{backend.py → _backend.py}

@@ -1,5 +1,3 @@
-import os
-
 # Define backend properties in a dictionary for easier maintenance
 
 BACKEND_PROPERTIES = {
@@ -49,14 +47,6 @@ BACKEND_PROPERTIES = {
         "default_password": None,
         "is_sqla_type": False,
     },
-    "redis": {
-        "uri_prefix": "redis://",
-        "default_port": 6379,
-        "default_host": "localhost",
-        "default_database": 0,
-        "default_username": None,
-        "default_password": None,
-    },
     "nats_kv": {
         "uri_prefix": "nats://",
         "default_port": 4222,
@@ -75,17 +65,6 @@ BACKEND_PROPERTIES = {
     },
 }
 
-# # REDIS ENVIRONMENT VARIABLES
-# REDIS_HOST = os.getenv("FP_REDIS_HOST", BACKEND_PROPERTIES["redis"]["default_host"])
-# REDIS_PORT = int(
-#     os.getenv("FP_REDIS_PORT", BACKEND_PROPERTIES["redis"]["default_port"])
-# )
-# REDIS_DB = int(
-#     os.getenv("FP_REDIS_DB", BACKEND_PROPERTIES["redis"]["default_database"])
-# )
-# REDIS_PASSWORD = os.getenv("FP_REDIS_PASSWORD", None)
-# REDIS_USERNAME = os.getenv("FP_REDIS_USERNAME", None)
-# REDIS_SSL = bool(os.getenv("FP_REDIS_SSL", False))
 
 # # POSTGRES ENVIRONMENT VARIABLES
 # POSTGRES_HOST = os.getenv(

flowerpower/settings/logging.py

@@ -1,4 +1,4 @@
 import os
 
 # LOGGING
-LOG_LEVEL = os.getenv("FP_LOG_LEVEL", "INFO")
+LOG_LEVEL = os.getenv("FP_LOG_LEVEL", "CRITICAL")

flowerpower/utils/logging.py

@@ -1,23 +1,35 @@
+import os
 import sys
 
 from loguru import logger
 
-from ..settings import LOG_LEVEL  # Import the setting
+from ..settings import LOG_LEVEL
 
 
-def setup_logging(level: str = LOG_LEVEL, disable: bool = False) -> None:
+def setup_logging(level: str | None = None) -> None:
     """
     Configures the Loguru logger.
 
-    Removes the default handler and adds a new one targeting stderr
-    with the level specified by the FP_LOG_LEVEL setting.
+    Determines the logging level based on the following precedence:
+    1. The 'level' argument passed to the function.
+    2. The 'FP_LOG_LEVEL' environment variable.
+    3. The 'LOG_LEVEL' from ..settings (which defaults to "CRITICAL").
+
+    If the effective logging level is "CRITICAL", logging for the "flowerpower" module
+    is disabled. Otherwise, logging is enabled and configured.
     """
-    logger.remove()  # Remove the default handler added by Loguru
-    logger.add(
-        sys.stderr,
-        level=level.upper(),  # Use the level from the parameter, ensure it's uppercase
-        format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",  # Example format
-    )
-    if disable:
+    # Remove all existing handlers to prevent duplicate logs
+    logger.remove()
+
+    # Determine the effective logging level
+    effective_level = level or os.getenv("FP_LOG_LEVEL") or LOG_LEVEL
+
+    if effective_level.upper() == "CRITICAL":
         logger.disable("flowerpower")
-    # logger.info(f"Log level set to: {FP_LOG_LEVEL.upper()}")
+    else:
+        logger.enable("flowerpower")
+        logger.add(
+            sys.stderr,
+            level=effective_level.upper(),
+            format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
+        )