FlowerPower 0.21.0__py3-none-any.whl → 0.30.0__py3-none-any.whl

flowerpower/cfg/pipeline/run.py

@@ -39,8 +39,34 @@ class RunConfig(BaseConfig):
     project_adapter_cfg: dict | None = msgspec.field(default=None)
     adapter: dict[str, Any] | None = msgspec.field(default=None)
     reload: bool = msgspec.field(default=False)
-    on_success: Callable | tuple[Callable, tuple | None, dict | None] | None = msgspec.field(default=None)
-    on_failure: Callable | tuple[Callable, tuple | None, dict | None] | None = msgspec.field(default=None)
+
+
+class CallbackSpec(msgspec.Struct):
+    """Specification for a callback function with optional arguments."""
+    func: Callable
+    args: tuple | None = None
+    kwargs: dict | None = None
+
+
+class RunConfig(BaseConfig):
+    inputs: dict | None = msgspec.field(default_factory=dict)
+    final_vars: list[str] | None = msgspec.field(default_factory=list)
+    config: dict | None = msgspec.field(default_factory=dict)
+    cache: dict | bool | None = msgspec.field(default=False)
+    with_adapter: WithAdapterConfig = msgspec.field(default_factory=WithAdapterConfig)
+    executor: ExecutorConfig = msgspec.field(default_factory=ExecutorConfig)
+    log_level: str | None = msgspec.field(default="INFO")
+    max_retries: int = msgspec.field(default=3)
+    retry_delay: int | float = msgspec.field(default=1)
+    jitter_factor: float | None = msgspec.field(default=0.1)
+    retry_exceptions: list[str] = msgspec.field(default_factory=lambda: ["Exception"])
+    # New fields for comprehensive configuration
+    pipeline_adapter_cfg: dict | None = msgspec.field(default=None)
+    project_adapter_cfg: dict | None = msgspec.field(default=None)
+    adapter: dict[str, Any] | None = msgspec.field(default=None)
+    reload: bool = msgspec.field(default=False)
+    on_success: CallbackSpec | None = msgspec.field(default=None)
+    on_failure: CallbackSpec | None = msgspec.field(default=None)
 
     def __post_init__(self):
         if isinstance(self.inputs, dict):
@@ -81,3 +107,30 @@ class RunConfig(BaseConfig):
                 else:
                     converted_exceptions.append(Exception)
             self.retry_exceptions = converted_exceptions
+
+        # Handle callback conversions
+        if self.on_success is not None and not isinstance(self.on_success, CallbackSpec):
+            if callable(self.on_success):
+                self.on_success = CallbackSpec(func=self.on_success)
+            elif isinstance(self.on_success, tuple) and len(self.on_success) == 3:
+                func, args, kwargs = self.on_success
+                self.on_success = CallbackSpec(func=func, args=args, kwargs=kwargs)
+            else:
+                self.on_success = None
+                warnings.warn(
+                    "Invalid on_success format, must be Callable or (Callable, args, kwargs)",
+                    RuntimeWarning
+                )
+
+        if self.on_failure is not None and not isinstance(self.on_failure, CallbackSpec):
+            if callable(self.on_failure):
+                self.on_failure = CallbackSpec(func=self.on_failure)
+            elif isinstance(self.on_failure, tuple) and len(self.on_failure) == 3:
+                func, args, kwargs = self.on_failure
+                self.on_failure = CallbackSpec(func=func, args=args, kwargs=kwargs)
+            else:
+                self.on_failure = None
+                warnings.warn(
+                    "Invalid on_failure format, must be Callable or (Callable, args, kwargs)",
+                    RuntimeWarning
+                )

flowerpower/cli/__init__.py

@@ -19,13 +19,6 @@ app.add_typer(
 )
 
 
-if importlib.util.find_spec("paho"):
-    from .mqtt import app as mqtt_app
-
-    app.add_typer(
-        mqtt_app, name="mqtt", help="Connect pipelines to MQTT message brokers"
-    )
-
 
 @app.command()
 def init(

flowerpower/pipeline/manager.py

@@ -40,13 +40,11 @@ class PipelineManager:
     - Configuration management and loading
     - Pipeline creation, deletion, and discovery
     - Pipeline execution via PipelineRunner
-    - Job scheduling via PipelineScheduler
     - Visualization via PipelineVisualizer
     - Import/export operations via PipelineIOManager
 
     Attributes:
         registry (PipelineRegistry): Handles pipeline registration and discovery
-        scheduler (PipelineScheduler): Manages job scheduling and execution
         visualizer (PipelineVisualizer): Handles pipeline visualization
         io (PipelineIOManager): Manages pipeline import/export operations
         project_cfg (ProjectConfig): Current project configuration

{flowerpower-0.21.0.dist-info → flowerpower-0.30.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.21.0
+Version: 0.30.0
 Summary: A simple workflow framework for building and managing data processing pipelines
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -11,7 +11,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duration-parser>=1.0.1
 Requires-Dist: fsspec>=2024.10.0
-Requires-Dist: fsspec-utils>=0.1.0
+Requires-Dist: fsspec-utils[full]>=0.1.0
 Requires-Dist: humanize>=4.12.2
 Requires-Dist: msgspec>=0.19.0
 Requires-Dist: munch>=4.0.0
@@ -25,26 +25,14 @@ Provides-Extra: io
 Requires-Dist: flowerpower-io>=0.1.1; extra == "io"
 Provides-Extra: io-legacy
 Requires-Dist: flowerpower-io[legacy]>=0.1.1; extra == "io-legacy"
-Provides-Extra: mongodb
-Requires-Dist: pymongo>=4.7.2; extra == "mongodb"
-Provides-Extra: mqtt
-Requires-Dist: paho-mqtt>=2.1.0; extra == "mqtt"
-Requires-Dist: orjson>=3.10.11; extra == "mqtt"
-Requires-Dist: mmh3>=5.1.0; extra == "mqtt"
 Provides-Extra: opentelemetry
 Requires-Dist: opentelemetry-api>=1.5.0; extra == "opentelemetry"
 Requires-Dist: opentelemetry-sdk>=1.5.0; extra == "opentelemetry"
 Requires-Dist: opentelemetry-exporter-jaeger>=1.21.0; extra == "opentelemetry"
 Provides-Extra: ray
 Requires-Dist: ray>=2.34.0; extra == "ray"
-Provides-Extra: tui
-Requires-Dist: textual>=0.85.2; extra == "tui"
 Provides-Extra: ui
 Requires-Dist: sf-hamilton-ui>=0.0.11; extra == "ui"
-Provides-Extra: webserver
-Requires-Dist: sanic>=24.6.0; extra == "webserver"
-Requires-Dist: sanic-ext>=23.12.0; extra == "webserver"
-Requires-Dist: orjson>=3.10.11; extra == "webserver"
 Provides-Extra: openlineage
 Requires-Dist: openlineage-python>=1.32.0; extra == "openlineage"
 Dynamic: license-file

{flowerpower-0.21.0.dist-info → flowerpower-0.30.0.dist-info}/RECORD

@@ -6,23 +6,21 @@ flowerpower/cfg/pipeline/__init__.py,sha256=HSUlDt9smo-zPqOljEl329CQg8qsd0EJfCH1
 flowerpower/cfg/pipeline/_schedule.py,sha256=kaDyij3eUj0u6VdmJW4x_AQbSAZ-r7s2Fk7QFkXU5JQ,1029
 flowerpower/cfg/pipeline/adapter.py,sha256=uBKV6BZlsRRqSYNyC1oEWPchsaH7rFPCBobG5BrF3ss,2265
 flowerpower/cfg/pipeline/builder.py,sha256=rZ-cspbV-nwtckvOA49vt7DNJNb-gRLHHWiejmJPsFs,13192
-flowerpower/cfg/pipeline/run.py,sha256=xdHkB3Dx2ejkRUeK9bGHrIcO_JkFH3Hok_AFSO0WWT8,4212
+flowerpower/cfg/pipeline/run.py,sha256=cirZbiHsY8D87idBvbfNv1gEtKpgv3xLfiiFIXJFxrA,6733
 flowerpower/cfg/project/__init__.py,sha256=bMMZ5Zo3YgkgOGt82H1mW8I4WnoCUpa0bSLPpZiB7K4,4560
 flowerpower/cfg/project/adapter.py,sha256=2k2U25NziDEiUacLTjxaSxOVkaQBRt6ECWGRGX0v4J0,1481
-flowerpower/cli/__init__.py,sha256=ZojMSbKvwtsOu0gryA5_GOEtUy3l6ZxYNvvKyhWsTxk,5079
+flowerpower/cli/__init__.py,sha256=RkJT3mPlSOi55dNNP8kpm_omF0zEfbmWVP16N-lkijE,4897
 flowerpower/cli/cfg.py,sha256=P7qEcjraitMxbzVWJMqWeitIdpUkW41QkUi7ol0ksW0,1455
-flowerpower/cli/mqtt.py,sha256=NuQtvKA3DrmLdGRsR7CFhbVhKZ4qObzwRlvENSkTc0k,6009
 flowerpower/cli/pipeline.py,sha256=I58cWjK7J4dkXdJXIePcPiD7iILnYqoje0ztkR5mNwg,21958
 flowerpower/cli/utils.py,sha256=tsxvKIqUhl4m9IzuaSoc5a3_gb6Fu4LYyV8fVodqIdA,5127
 flowerpower/pipeline/__init__.py,sha256=ltr4LQnM5Boa9w7t-lNVmmcSglIkTPuIoeYnEKWU4Og,122
 flowerpower/pipeline/base.py,sha256=oQSDfEAyY2_kDRlHNnHr6ihZvfGeOjyMJRPKob1i7U8,3560
 flowerpower/pipeline/io.py,sha256=phYJhN4LZ0c6d8_udEQ4C9cGzeV3Ku0hsj0gyE1n2UY,16246
-flowerpower/pipeline/manager.py,sha256=ILnFUI8YeyCJsu7wS8g8Os6cNIhxoK6EihfkN5azWaI,47698
+flowerpower/pipeline/manager.py,sha256=gJMWe_T1WpxO2H9GDrNtf61S4oJuLugMHwX82jqAuEU,47579
 flowerpower/pipeline/pipeline.py,sha256=8Vhc3GGoAJI-5zv8qX4gZ5JwJrTdVoiXHWNpv5OeKGw,26207
 flowerpower/pipeline/registry.py,sha256=iHdEBPiRpZorAUe6sDkqONoP8t6cicdQYGdK8UyxuAQ,28791
 flowerpower/pipeline/visualizer.py,sha256=EVpjv-TUe1zGvdEAWyShJcVXurm02W0jkLbj7z1uAv4,4953
 flowerpower/plugins/io/__init__.py,sha256=ZmSdKoh3TavJagOz0vUItnEqAh3mAM1QpAWj0KufF_k,222
-flowerpower/plugins/mqtt/__init__.py,sha256=4eTTvYY4SjC7O2lMaK-Hw5LQGvZE8TVl6HsXuVVD9zQ,228
 flowerpower/settings/__init__.py,sha256=XKQa8AI9VrX8ievs-suq3Cm6PBt4cJ78ZHVIjUbXCyA,130
 flowerpower/settings/_backend.py,sha256=Up1RBqAs3jtDUOV-9wEpL68Qmom-dRWMOeHXIh0F3lw,4273
 flowerpower/settings/executor.py,sha256=vNF383g1gMGkq_CXUzateGMNcJZig-vHkVVb0Hi_b74,249
@@ -36,9 +34,9 @@ flowerpower/utils/misc.py,sha256=gY1KVJp4lF4S0THUu1LUrjXaf87TzLO-TcwHxZZh2Zk,841
 flowerpower/utils/monkey.py,sha256=vJMYANjZI13PNbEQThdX0EFP1_6bGNHgnpF7HwReNyM,58
 flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
 flowerpower/utils/templates.py,sha256=ouyEeSDqa9PjW8c32fGpcINlpC0WToawRFZkMPtwsLE,1591
-flowerpower-0.21.0.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
-flowerpower-0.21.0.dist-info/METADATA,sha256=fkVSaKr-aU6eiaRAvydDHWfXf1J_a02Ca8Zw_zCjgHA,17691
-flowerpower-0.21.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flowerpower-0.21.0.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
-flowerpower-0.21.0.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
-flowerpower-0.21.0.dist-info/RECORD,,
+flowerpower-0.30.0.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
+flowerpower-0.30.0.dist-info/METADATA,sha256=3nPehMWbDcryhbkgbieU-_13I_8S3-1aNL54bMdjdDs,17208
+flowerpower-0.30.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flowerpower-0.30.0.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
+flowerpower-0.30.0.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
+flowerpower-0.30.0.dist-info/RECORD,,

flowerpower/cli/mqtt.py

@@ -1,168 +0,0 @@
-import importlib
-import sys
-
-import typer
-
-from ..plugins.mqtt import run_pipeline_on_message as run_pipeline_on_message_
-from ..plugins.mqtt import start_listener as start_listener_
-from .utils import load_hook, parse_dict_or_list_param
-
-app = typer.Typer(help="MQTT management commands")
-
-
-@app.command()
-def start_listener(
-    on_message: str,
-    topic: str,
-    base_dir: str,
-    host: str = "localhost",
-    port: int = 1883,
-    username: str | None = None,
-    password: str | None = None,
-):
-    """Start an MQTT client to listen to messages on a topic
-
-    The connection to the MQTT broker is established using the provided configuration o a
-    MQTT event broker defined in the project configuration file `conf/project.yml`.
-    If not configuration is found, you have to provide the connection parameters,
-    such as `host`, `port`, `username`, and `password`.
-
-    The `on_message` module should contain a function `on_message` that will be called
-    with the message payload as argument.
-
-    Args:
-        on_message: Name of the module containing the on_message function
-        topic: MQTT topic to listen to
-        base_dir: Base directory for the module
-        host: MQTT broker host
-        port: MQTT broker port
-        username: MQTT broker username
-        password: MQTT broker password
-
-    Examples:
-        $ flowerpower mqtt start_listener --on-message my_module --topic my_topic --base-dir /path/to/module
-    """
-    sys.path.append(base_dir)
-    on_message_module = importlib.import_module(on_message)
-    start_listener_(
-        on_message=on_message_module.on_message,
-        topic=topic,
-        base_dir=base_dir,
-        host=host,
-        port=port,
-        username=username,
-        password=password,
-        background=False,
-    )
-
-
-@app.command()
-def run_pipeline_on_message(
-    name: str,
-    topic: str | None = None,
-    executor: str | None = None,
-    base_dir: str | None = None,
-    inputs: str | None = None,
-    final_vars: str | None = None,
-    config: str | None = None,
-    with_tracker: bool = False,
-    with_opentelemetry: bool = False,
-    with_progressbar: bool = False,
-    storage_options: str | None = None,
-    host: str | None = None,
-    port: int | None = None,
-    username: str | None = None,
-    password: str | None = None,
-    clean_session: bool = True,
-    qos: int = 0,
-    client_id: str | None = None,
-    client_id_suffix: str | None = None,
-    config_hook: str | None = None,
-    max_retries: int = typer.Option(
-        3, help="Maximum number of retry attempts if pipeline execution fails"
-    ),
-    retry_delay: float = typer.Option(
-        1.0, help="Base delay between retries in seconds"
-    ),
-    jitter_factor: float = typer.Option(
-        0.1, help="Random factor (0-1) applied to delay for jitter"
-    ),
-):
-    """Run a pipeline on a message
-
-    This command sets up an MQTT listener that executes a pipeline whenever a message is
-    received on the specified topic. The pipeline can be configured to retry on failure
-    using exponential backoff with jitter for better resilience.
-
-    Args:
-        name: Name of the pipeline
-        topic: MQTT topic to listen to
-        executor: Name of the executor
-        base_dir: Base directory for the pipeline
-        inputs: Inputs as JSON or key=value pairs or dict string
-        final_vars: Final variables as JSON or list
-        config: Config for the hamilton pipeline executor
-        with_tracker: Enable tracking with hamilton ui
-        with_opentelemetry: Enable OpenTelemetry tracing
-        with_progressbar: Enable progress bar
-        storage_options: Storage options as JSON, dict string or key=value pairs
-        host: MQTT broker host
-        port: MQTT broker port
-        username: MQTT broker username
-        password: MQTT broker password
-        clean_session: Whether to start a clean session with the broker
-        qos: MQTT Quality of Service level (0, 1, or 2)
-        client_id: Custom MQTT client identifier
-        client_id_suffix: Optional suffix to append to client_id
-        config_hook: Function to process incoming messages into pipeline config
-        max_retries: Maximum number of retry attempts if pipeline execution fails
-        retry_delay: Base delay between retries in seconds
-        jitter_factor: Random factor (0-1) applied to delay for jitter
-
-    Examples:
-        # Basic usage with a specific topic
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic sensors/data
-
-        # Configure retries for resilience
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic sensors/data --max-retries 5 --retry-delay 2.0
-
-        # Use a config hook to process messages
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic data/incoming --config-hook process_message
-
-
-    """
-
-    parsed_inputs = parse_dict_or_list_param(inputs, "dict")
-    parsed_config = parse_dict_or_list_param(config, "dict")
-    parsed_final_vars = parse_dict_or_list_param(final_vars, "list")
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict")
-
-    config_hook_function = None
-    if config_hook:
-        config_hook_function = load_hook(name, config_hook, base_dir, storage_options)
-
-    run_pipeline_on_message_(
-        name=name,
-        topic=topic,
-        executor=executor,
-        base_dir=base_dir,
-        inputs=parsed_inputs,
-        final_vars=parsed_final_vars,
-        config=parsed_config,
-        with_tracker=with_tracker,
-        with_opentelemetry=with_opentelemetry,
-        with_progressbar=with_progressbar,
-        storage_options=parsed_storage_options,
-        host=host,
-        port=port,
-        username=username,
-        password=password,
-        clean_session=clean_session,
-        qos=qos,
-        client_id=client_id,
-        client_id_suffix=client_id_suffix,
-        config_hook=config_hook_function,
-        max_retries=max_retries,
-        retry_delay=retry_delay,
-        jitter_factor=jitter_factor,
-    )

flowerpower/plugins/mqtt/__init__.py

@@ -1,8 +0,0 @@
-import warnings
-
-warnings.warn(
-    "The flowerpower.plugins.mqtt module is deprecated. "
-    "Please use 'flowerpower-mqtt' instead. Install it with 'pip install flowerpower-mqtt'.",
-    DeprecationWarning,
-    stacklevel=2,
-)