FlowerPower 1.0.0b2__py3-none-any.whl → 1.0.0b4__py3-none-any.whl

flowerpower/pipeline/manager.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from types import TracebackType
 from typing import Any, Callable, TypeVar, Union
 from uuid import UUID
+
 import duration_parser
 from loguru import logger
 from munch import Munch
@@ -22,9 +23,9 @@ from ..cfg.project.adapter import AdapterConfig as ProjectAdapterConfig
 from ..fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
 from ..utils.logging import setup_logging
 from .io import PipelineIOManager
-from .registry import PipelineRegistry, HookType
-from .runner import PipelineRunner, run_pipeline
 from .job_queue import PipelineJobQueue
+from .registry import HookType, PipelineRegistry
+from .runner import PipelineRunner, run_pipeline
 from .visualizer import PipelineVisualizer
 
 setup_logging()
@@ -416,7 +417,6 @@ class PipelineManager:
         retry_delay: float | None = None,
         jitter_factor: float | None = None,
         retry_exceptions: tuple | list | None = None,
-
     ) -> dict[str, Any]:
         """Execute a pipeline synchronously and return its results.
 
@@ -495,7 +495,7 @@ class PipelineManager:
             pipeline_adapter_cfg=pipeline_adapter_cfg,
             project_adapter_cfg=project_adapter_cfg,
             adapter=adapter,
-            #reload=reload,  # Runner handles module reload if needed
+            # reload=reload,  # Runner handles module reload if needed
             log_level=log_level,
             max_retries=max_retries,
             retry_delay=retry_delay,
@@ -649,7 +649,7 @@ class PipelineManager:
             >>>
             >>> manager = PipelineManager()
             >>> manager.show_pipelines()
-           
+
         """
         self.registry.show_pipelines()
 
@@ -706,21 +706,25 @@ class PipelineManager:
             ml_pipeline: streaming
         """
         return self.registry.summary
-    
+
     def add_hook(
-            self, name:str, type:HookType, to:str|None, function_name:str|None,
-    )->None:
+        self,
+        name: str,
+        type: HookType,
+        to: str | None,
+        function_name: str | None,
+    ) -> None:
         """Add a hook to the pipeline module.
-        
+
         Args:
             name (str): The name of the pipeline
             type (HookType): The type of the hook.
             to (str | None, optional): The name of the file to add the hook to. Defaults to the hook.py file in the pipelines hooks folder.
             function_name (str | None, optional): The name of the function. If not provided uses default name of hook type.
-            
+
         Returns:
             None
-        
+
         Raises:
             ValueError: If the hook type is not valid
 
@@ -734,7 +738,7 @@ class PipelineManager:
             ...     to="pre_execute_hook",
             ...     function_name="my_pre_execute_function"
             ... )
-            """
+        """
         self.registry.add_hook(
             name=name,
             type=type,
@@ -748,7 +752,7 @@ class PipelineManager:
         name: str,
         base_dir: str,
         src_fs: AbstractFileSystem | None = None,
-        storage_options: BaseStorageOptions | None = None,
+        storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ) -> None:
         """Import a pipeline from another FlowerPower project.
@@ -807,7 +811,7 @@ class PipelineManager:
         pipelines: dict[str, str] | list[str],
         base_dir: str,  # Base dir for source if pipelines is a list
         src_fs: AbstractFileSystem | None = None,
-        src_storage_options: BaseStorageOptions | None = None,
+        src_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ) -> None:
         """Import multiple pipelines from another project or location.
@@ -867,7 +871,7 @@ class PipelineManager:
         self,
         base_dir: str,
         src_fs: AbstractFileSystem | None = None,
-        src_storage_options: BaseStorageOptions | None = None,
+        src_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ) -> None:
         """Import all pipelines from another FlowerPower project.
@@ -911,7 +915,7 @@ class PipelineManager:
         name: str,
         base_dir: str,
         dest_fs: AbstractFileSystem | None = None,
-        dest_storage_options: BaseStorageOptions | None = None,
+        dest_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ) -> None:
         """Export a pipeline to another location or project.
@@ -970,7 +974,7 @@ class PipelineManager:
         pipelines: list[str],
         base_dir: str,
         dest_fs: AbstractFileSystem | None = None,
-        dest_storage_options: BaseStorageOptions | None = None,
+        dest_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ) -> None:
         """Export multiple pipelines to another location.
@@ -1024,7 +1028,7 @@ class PipelineManager:
         self,
         base_dir: str,
         dest_fs: AbstractFileSystem | None = None,
-        dest_storage_options: BaseStorageOptions | None = None,
+        dest_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ) -> None:
         """Export all pipelines to another location.
@@ -1206,7 +1210,7 @@ class PipelineManager:
             retry_delay (float): Delay between retries in seconds.
             jitter_factor (float): Random jitter factor to add to retry delay
             retry_exceptions (tuple): Exceptions that trigger a retry.
-            
+
             **kwargs: JobQueue-specific arguments
                 For RQ:
                     - queue_name: Queue to use (str)
@@ -1252,7 +1256,7 @@ class PipelineManager:
             pipeline_adapter_cfg=pipeline_adapter_cfg,
             project_adapter_cfg=project_adapter_cfg,
             adapter=adapter,
-            #reload=reload,
+            # reload=reload,
             log_level=log_level,
             max_retries=max_retries,
             retry_delay=retry_delay,
@@ -1277,7 +1281,7 @@ class PipelineManager:
         log_level: str | None = None,
         result_ttl: int | dt.timedelta = 0,
         run_at: dt.datetime | str | None = None,
-        run_in: dt.datetime |  str | None = None,
+        run_in: dt.datetime | str | None = None,
         max_retries: int = 3,
         retry_delay: float = 1.0,
         jitter_factor: float = 0.1,
@@ -1345,9 +1349,12 @@ class PipelineManager:
 
         """
         run_func = self._get_run_func_for_job(name, reload)
-        run_in = duration_parser.parse(run_in) if isinstance(run_in, str) else run_in #convert to seconds
-        run_at = dt.datetime.fromisoformat(run_at) if isinstance(run_at, str) else run_at
-
+        run_in = (
+            duration_parser.parse(run_in) if isinstance(run_in, str) else run_in
+        )  # convert to seconds
+        run_at = (
+            dt.datetime.fromisoformat(run_at) if isinstance(run_at, str) else run_at
+        )
 
         return self.job_queue.add_job(
             run_func=run_func,
@@ -1362,7 +1369,7 @@ class PipelineManager:
             pipeline_adapter_cfg=pipeline_adapter_cfg,
             project_adapter_cfg=project_adapter_cfg,
             adapter=adapter,
-            #reload=reload,  # Note: reload already happened
+            # reload=reload,  # Note: reload already happened
             log_level=log_level,
             result_ttl=result_ttl,
             run_at=run_at,
@@ -1476,7 +1483,9 @@ class PipelineManager:
         """
         pipeline_cfg = self._load_pipeline_cfg(name=name, reload=reload)
         run_func = self._get_run_func_for_job(name, reload)
-        interval = duration_parser.parse(interval) if isinstance(interval, str) else interval
+        interval = (
+            duration_parser.parse(interval) if isinstance(interval, str) else interval
+        )
         date = dt.datetime.fromisoformat(date) if isinstance(date, str) else date
 
         return self.job_queue.schedule(
@@ -1583,4 +1592,3 @@ class PipelineManager:
         except Exception as e:
             logger.error(f"Failed to retrieve schedules: {e}")
             return []
-

flowerpower/pipeline/registry.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 """Pipeline Registry for discovery, listing, creation, and deletion."""
-import os
+
 import datetime as dt
+import os
 import posixpath
 from typing import TYPE_CHECKING
 
@@ -14,12 +15,14 @@ from rich.table import Table
 from rich.tree import Tree
 
 from .. import settings
+
 # Import necessary config types and utility functions
 from ..cfg import PipelineConfig, ProjectConfig
 from ..fs import AbstractFileSystem
 from ..utils.logging import setup_logging
+
 # Assuming view_img might be used indirectly or needed later
-from ..utils.templates import PIPELINE_PY_TEMPLATE, HOOK_TEMPLATE__MQTT_BUILD_CONFIG
+from ..utils.templates import HOOK_TEMPLATE__MQTT_BUILD_CONFIG, PIPELINE_PY_TEMPLATE
 
 if TYPE_CHECKING:
     # Keep this for type hinting if needed elsewhere, though Config is imported directly now
@@ -27,6 +30,7 @@ if TYPE_CHECKING:
 
 from enum import Enum
 
+
 class HookType(str, Enum):
     MQTT_BUILD_CONFIG = "mqtt-build-config"
 
@@ -40,6 +44,7 @@ class HookType(str, Enum):
     def __str__(self) -> str:
         return self.value
 
+
 setup_logging(level=settings.LOG_LEVEL)
 
 
@@ -443,12 +448,14 @@ class PipelineRegistry:
                 logger.warning(f"Could not get size for {path}: {e}")
                 size = "Error"
 
-            pipeline_info.append({
-                "name": name,
-                "path": path,
-                "mod_time": mod_time,
-                "size": size,
-            })
+            pipeline_info.append(
+                {
+                    "name": name,
+                    "path": path,
+                    "mod_time": mod_time,
+                    "size": size,
+                }
+            )
 
         if show:
             table = Table(title="Available Pipelines")
@@ -514,7 +521,13 @@ class PipelineRegistry:
         """
         return self._all_pipelines(show=False)
 
-    def add_hook(self, name: str, type: HookType, to: str | None = None, function_name: str|None = None):
+    def add_hook(
+        self,
+        name: str,
+        type: HookType,
+        to: str | None = None,
+        function_name: str | None = None,
+    ):
         """
         Add a hook to the pipeline module.
 
@@ -534,7 +547,6 @@ class PipelineRegistry:
             ```
         """
 
-        
         if to is None:
             to = f"hooks/{name}/hook.py"
         else:
@@ -551,10 +563,8 @@ class PipelineRegistry:
             self._fs.makedirs(os.path.dirname(to), exist_ok=True)
 
         with self._fs.open(to, "a") as f:
-            f.write(
-                template.format(
-                function_name=function_name
-                )
-            )
+            f.write(template.format(function_name=function_name))
 
-        rich.print(f"🔧 Added hook [bold blue]{type.value}[/bold blue] to {to} as {function_name} for {name}")    
+        rich.print(
+            f"🔧 Added hook [bold blue]{type.value}[/bold blue] to {to} as {function_name} for {name}"
+        )

flowerpower/pipeline/runner.py

@@ -2,10 +2,11 @@
 """Pipeline Runner."""
 
 from __future__ import annotations
-import time
-import random
+
 import datetime as dt
 import importlib.util
+import random
+import time
 from typing import Any, Callable
 
 import humanize
@@ -14,12 +15,10 @@ from hamilton.execution import executors
 from hamilton.registry import disable_autoload
 from hamilton.telemetry import disable_telemetry
 from hamilton_sdk.api.clients import UnauthorizedException
-
 from requests.exceptions import HTTPError
 
 from .. import settings
 
-
 if importlib.util.find_spec("opentelemetry"):
     from hamilton.plugins import h_opentelemetry
 

flowerpower/plugins/mqtt/__init__.py

@@ -1,12 +1,12 @@
 from .cfg import MqttConfig
-from .manager import MqttManager, start_listener, run_pipeline_on_message
+from .manager import MqttManager, run_pipeline_on_message, start_listener
 
 MQTTManager = MqttManager
 
 __all__ = [
-    'MqttConfig',
-    'MqttManager',
-    'MQTTManager',
-    'start_listener',
-    'run_pipeline_on_message',
-]
+    "MqttConfig",
+    "MqttManager",
+    "MQTTManager",
+    "start_listener",
+    "run_pipeline_on_message",
+]

flowerpower/plugins/mqtt/cfg.py

@@ -1,16 +1,17 @@
 from ...cfg.base import BaseConfig
 
+
 class MqttConfig(BaseConfig):
     username: str | None = None
     password: str | None = None
-    host: str | None = 'localhost'
+    host: str | None = "localhost"
     port: int | None = 1883
     topic: str | None = None
     first_reconnect_delay: int = 1
     max_reconnect_count: int = 5
     reconnect_rate: int = 2
     max_reconnect_delay: int = 60
-    transport: str = 'tcp'
+    transport: str = "tcp"
     clean_session: bool = True
     client_id: str | None = None
     client_id_suffix: str | None = None

flowerpower/plugins/mqtt/manager.py

@@ -4,20 +4,20 @@ import socket
 import time
 from pathlib import Path
 from types import TracebackType
-from typing import Callable, Any
+from typing import Any, Callable
 
 import mmh3
 from loguru import logger
 from munch import Munch
 from paho.mqtt.client import CallbackAPIVersion, Client
 
-from .cfg import MqttConfig
 from ...cfg import ProjectConfig
 from ...cfg.pipeline.run import ExecutorConfig, WithAdapterConfig
 from ...cfg.project.adapter import AdapterConfig
+from ...fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
 from ...pipeline.manager import PipelineManager
 from ...utils.logging import setup_logging
-from ...fs import get_filesystem, AbstractFileSystem, BaseStorageOptions
+from .cfg import MqttConfig
 
 setup_logging()
 
@@ -89,28 +89,37 @@ class MqttManager:
         else:
             event_broker_cfg = jq_backend.event_broker
             return cls(
-               **event_broker_cfg.dict(),
+                **event_broker_cfg.dict(),
             )
 
     @classmethod
-    def from_config(cls, cfg: MqttConfig | None=None, path: str | None = None, fs: AbstractFileSystem | None = None, storage_options: dict | BaseStorageOptions = {}):
+    def from_config(
+        cls,
+        cfg: MqttConfig | None = None,
+        path: str | None = None,
+        fs: AbstractFileSystem | None = None,
+        storage_options: dict | BaseStorageOptions = {},
+    ):
         if cfg is None:
             if path is None:
                 raise ValueError(
                     "No configuration provided. Please provide `config` or `path` to the configuration file."
                 )
-            
+
         if cfg is None:
             import os
+
             if fs is None:
-                fs = get_filesystem(path=os.path.dirname(path), storage_options=storage_options)
+                fs = get_filesystem(
+                    path=os.path.dirname(path), storage_options=storage_options
+                )
 
             cfg = MqttConfig.from_yaml(path=os.path.basename(path), fs=fs)
 
         return cls(
             **cfg.dict(),
         )
-    
+
     @classmethod
     def from_dict(cls, cfg: dict):
         return cls(
@@ -344,8 +353,6 @@ class MqttManager:
         self._client.loop_stop()
         logger.info("Client stopped.")
 
-    
-
     def run_pipeline_on_message(
         self,
         name: str,
@@ -447,12 +454,9 @@ class MqttManager:
                 config_ = config_hook(inputs["payload"], inputs["topic"])
                 logger.debug(f"Config from hook: {config_}")
                 if any([k in config_ for k in config.keys()]):
-                    logger.warning(
-                        "Config from hook overwrites config from pipeline"
-                    )
+                    logger.warning("Config from hook overwrites config from pipeline")
                 config.update(config_)
                 logger.debug(f"Config after update: {config}")
-                
 
             with PipelineManager(
                 storage_options=storage_options, fs=fs, base_dir=base_dir
@@ -478,7 +482,7 @@ class MqttManager:
                             retry_delay=retry_delay,
                             jitter_factor=jitter_factor,
                             retry_exceptions=retry_exceptions,
-                            **kwargs,  
+                            **kwargs,
                         )
                     else:
                         pipeline.run(
@@ -564,7 +568,7 @@ def start_listener(
     Raises:
         ValueError: If the config_hook is not callable
         ValueError: If no client configuration is found
-    
+
     Example:
         ```python
         from flowerpower.plugins.mqtt import start_listener
@@ -596,7 +600,6 @@ def start_listener(
                 client_id_suffix=client_id_suffix,
                 config_hook=config_hook,
                 **kwargs,
-                
             )
         else:
             raise ValueError(
@@ -605,7 +608,9 @@ def start_listener(
                 "configured in the `config/project.yml` file."
             )
 
-    client.start_listener(on_message=on_message, topic=topic, background=background, qos=qos)
+    client.start_listener(
+        on_message=on_message, topic=topic, background=background, qos=qos
+    )
 
 
 def run_pipeline_on_message(
@@ -688,7 +693,7 @@ def run_pipeline_on_message(
 
     Returns:
         None
-    
+
     Raises:
         ValueError: If the config_hook is not callable
         ValueError: If no client configuration is found
@@ -723,9 +728,8 @@ def run_pipeline_on_message(
                 clean_session=clean_session,
                 client_id=client_id,
                 client_id_suffix=client_id_suffix,
-                config_hook=config_hook,                
+                config_hook=config_hook,
                 **kwargs,
-                
             )
         else:
             raise ValueError(
@@ -785,5 +789,3 @@ def run_pipeline_on_message(
         config_hook=config_hook,
         **kwargs,
     )
-
-

flowerpower/utils/misc.py

@@ -153,10 +153,12 @@ if importlib.util.find_spec("polars"):
                     next(v for v in data.values() if isinstance(v, (list, tuple)))
                 )
                 # Convert to DataFrame where each list element becomes a row
-                data = pl.DataFrame({
-                    k: v if isinstance(v, (list, tuple)) else [v] * length
-                    for k, v in data.items()
-                })
+                data = pl.DataFrame(
+                    {
+                        k: v if isinstance(v, (list, tuple)) else [v] * length
+                        for k, v in data.items()
+                    }
+                )
             else:
                 # If values are scalars, wrap them in a list to create a single row
                 data = pl.DataFrame({k: [v] for k, v in data.items()})

flowerpower/utils/templates.py

@@ -1,6 +1,3 @@
-
-
-
 PIPELINE_PY_TEMPLATE = """# FlowerPower pipeline {name}.py
 # Created on {date}
 
@@ -47,4 +44,4 @@ def {function_name}(payload: bytes, topic: str) -> dict:
     """
 
     pass
-'''
+'''

{flowerpower-1.0.0b2.dist-info → flowerpower-1.0.0b4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 1.0.0b2
+Version: 1.0.0b4
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower