FlowerPower 0.21.0__tar.gz → 0.30.0__tar.gz

{flowerpower-0.21.0/src/FlowerPower.egg-info → flowerpower-0.30.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.21.0
+Version: 0.30.0
 Summary: A simple workflow framework for building and managing data processing pipelines
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -11,7 +11,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duration-parser>=1.0.1
 Requires-Dist: fsspec>=2024.10.0
-Requires-Dist: fsspec-utils>=0.1.0
+Requires-Dist: fsspec-utils[full]>=0.1.0
 Requires-Dist: humanize>=4.12.2
 Requires-Dist: msgspec>=0.19.0
 Requires-Dist: munch>=4.0.0
@@ -25,26 +25,14 @@ Provides-Extra: io
 Requires-Dist: flowerpower-io>=0.1.1; extra == "io"
 Provides-Extra: io-legacy
 Requires-Dist: flowerpower-io[legacy]>=0.1.1; extra == "io-legacy"
-Provides-Extra: mongodb
-Requires-Dist: pymongo>=4.7.2; extra == "mongodb"
-Provides-Extra: mqtt
-Requires-Dist: paho-mqtt>=2.1.0; extra == "mqtt"
-Requires-Dist: orjson>=3.10.11; extra == "mqtt"
-Requires-Dist: mmh3>=5.1.0; extra == "mqtt"
 Provides-Extra: opentelemetry
 Requires-Dist: opentelemetry-api>=1.5.0; extra == "opentelemetry"
 Requires-Dist: opentelemetry-sdk>=1.5.0; extra == "opentelemetry"
 Requires-Dist: opentelemetry-exporter-jaeger>=1.21.0; extra == "opentelemetry"
 Provides-Extra: ray
 Requires-Dist: ray>=2.34.0; extra == "ray"
-Provides-Extra: tui
-Requires-Dist: textual>=0.85.2; extra == "tui"
 Provides-Extra: ui
 Requires-Dist: sf-hamilton-ui>=0.0.11; extra == "ui"
-Provides-Extra: webserver
-Requires-Dist: sanic>=24.6.0; extra == "webserver"
-Requires-Dist: sanic-ext>=23.12.0; extra == "webserver"
-Requires-Dist: orjson>=3.10.11; extra == "webserver"
 Provides-Extra: openlineage
 Requires-Dist: openlineage-python>=1.32.0; extra == "openlineage"
 Dynamic: license-file

{flowerpower-0.21.0 → flowerpower-0.30.0}/pyproject.toml

@@ -4,21 +4,14 @@ description = "A simple workflow framework for building and managing data proces
 authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
 readme = "README.md"
 requires-python = ">= 3.11"
-version = "0.21.0"
-keywords = [
-  "hamilton",
-  "workflow",
-  "pipeline",
-  "scheduler",
-  "dask",
-  "ray",
-]
+version = "0.30.0"
+keywords = ["hamilton", "workflow", "pipeline", "scheduler", "dask", "ray"]
 
 dependencies = [
   #'dill>=0.3.8',
   'duration-parser>=1.0.1',
   'fsspec>=2024.10.0',
-  'fsspec-utils>=0.1.0',
+  'fsspec-utils[full]>=0.1.0',
   'humanize>=4.12.2',
   'msgspec>=0.19.0',
   'munch>=4.0.0',
@@ -44,17 +37,14 @@ flowerpower = "flowerpower.cli:app"
 [project.optional-dependencies]
 io = ["flowerpower-io>=0.1.1"]
 io-legacy = ["flowerpower-io[legacy]>=0.1.1"]
-mongodb = ["pymongo>=4.7.2"]
-mqtt = ["paho-mqtt>=2.1.0", "orjson>=3.10.11", "mmh3>=5.1.0"]
 opentelemetry = [
   "opentelemetry-api>=1.5.0",
   "opentelemetry-sdk>=1.5.0",
   "opentelemetry-exporter-jaeger>=1.21.0", #"sf-hamilton[opentelemetry]>=1.83.3"
 ]
 ray = ["ray>=2.34.0"]
-tui = ["textual>=0.85.2"]
 ui = ["sf-hamilton-ui>=0.0.11"]
-webserver = ["sanic>=24.6.0", "sanic-ext>=23.12.0", "orjson>=3.10.11"]
+
 openlineage = ["openlineage-python>=1.32.0"]
 
 
@@ -77,5 +67,6 @@ dev-dependencies = [
   "pymdown-extensions>=10.16.1",
   "mkdocstrings>=0.30.0",
   "mkdocstrings-python>=1.17.0",
+  "repomix>=0.3.4",
 ]
 package = true

{flowerpower-0.21.0 → flowerpower-0.30.0/src/FlowerPower.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.21.0
+Version: 0.30.0
 Summary: A simple workflow framework for building and managing data processing pipelines
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -11,7 +11,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duration-parser>=1.0.1
 Requires-Dist: fsspec>=2024.10.0
-Requires-Dist: fsspec-utils>=0.1.0
+Requires-Dist: fsspec-utils[full]>=0.1.0
 Requires-Dist: humanize>=4.12.2
 Requires-Dist: msgspec>=0.19.0
 Requires-Dist: munch>=4.0.0
@@ -25,26 +25,14 @@ Provides-Extra: io
 Requires-Dist: flowerpower-io>=0.1.1; extra == "io"
 Provides-Extra: io-legacy
 Requires-Dist: flowerpower-io[legacy]>=0.1.1; extra == "io-legacy"
-Provides-Extra: mongodb
-Requires-Dist: pymongo>=4.7.2; extra == "mongodb"
-Provides-Extra: mqtt
-Requires-Dist: paho-mqtt>=2.1.0; extra == "mqtt"
-Requires-Dist: orjson>=3.10.11; extra == "mqtt"
-Requires-Dist: mmh3>=5.1.0; extra == "mqtt"
 Provides-Extra: opentelemetry
 Requires-Dist: opentelemetry-api>=1.5.0; extra == "opentelemetry"
 Requires-Dist: opentelemetry-sdk>=1.5.0; extra == "opentelemetry"
 Requires-Dist: opentelemetry-exporter-jaeger>=1.21.0; extra == "opentelemetry"
 Provides-Extra: ray
 Requires-Dist: ray>=2.34.0; extra == "ray"
-Provides-Extra: tui
-Requires-Dist: textual>=0.85.2; extra == "tui"
 Provides-Extra: ui
 Requires-Dist: sf-hamilton-ui>=0.0.11; extra == "ui"
-Provides-Extra: webserver
-Requires-Dist: sanic>=24.6.0; extra == "webserver"
-Requires-Dist: sanic-ext>=23.12.0; extra == "webserver"
-Requires-Dist: orjson>=3.10.11; extra == "webserver"
 Provides-Extra: openlineage
 Requires-Dist: openlineage-python>=1.32.0; extra == "openlineage"
 Dynamic: license-file

{flowerpower-0.21.0 → flowerpower-0.30.0}/src/FlowerPower.egg-info/SOURCES.txt

@@ -20,7 +20,6 @@ src/flowerpower/cfg/project/__init__.py
 src/flowerpower/cfg/project/adapter.py
 src/flowerpower/cli/__init__.py
 src/flowerpower/cli/cfg.py
-src/flowerpower/cli/mqtt.py
 src/flowerpower/cli/pipeline.py
 src/flowerpower/cli/utils.py
 src/flowerpower/pipeline/__init__.py
@@ -31,7 +30,6 @@ src/flowerpower/pipeline/pipeline.py
 src/flowerpower/pipeline/registry.py
 src/flowerpower/pipeline/visualizer.py
 src/flowerpower/plugins/io/__init__.py
-src/flowerpower/plugins/mqtt/__init__.py
 src/flowerpower/settings/__init__.py
 src/flowerpower/settings/_backend.py
 src/flowerpower/settings/executor.py

{flowerpower-0.21.0 → flowerpower-0.30.0}/src/FlowerPower.egg-info/requires.txt

@@ -1,6 +1,6 @@
 duration-parser>=1.0.1
 fsspec>=2024.10.0
-fsspec-utils>=0.1.0
+fsspec-utils[full]>=0.1.0
 humanize>=4.12.2
 msgspec>=0.19.0
 munch>=4.0.0
@@ -17,14 +17,6 @@ flowerpower-io>=0.1.1
 [io-legacy]
 flowerpower-io[legacy]>=0.1.1
 
-[mongodb]
-pymongo>=4.7.2
-
-[mqtt]
-paho-mqtt>=2.1.0
-orjson>=3.10.11
-mmh3>=5.1.0
-
 [openlineage]
 openlineage-python>=1.32.0
 
@@ -36,13 +28,5 @@ opentelemetry-exporter-jaeger>=1.21.0
 [ray]
 ray>=2.34.0
 
-[tui]
-textual>=0.85.2
-
 [ui]
 sf-hamilton-ui>=0.0.11
-
-[webserver]
-sanic>=24.6.0
-sanic-ext>=23.12.0
-orjson>=3.10.11

{flowerpower-0.21.0 → flowerpower-0.30.0}/src/flowerpower/cfg/pipeline/run.py

@@ -39,8 +39,34 @@ class RunConfig(BaseConfig):
     project_adapter_cfg: dict | None = msgspec.field(default=None)
     adapter: dict[str, Any] | None = msgspec.field(default=None)
     reload: bool = msgspec.field(default=False)
-    on_success: Callable | tuple[Callable, tuple | None, dict | None] | None = msgspec.field(default=None)
-    on_failure: Callable | tuple[Callable, tuple | None, dict | None] | None = msgspec.field(default=None)
+
+
+class CallbackSpec(msgspec.Struct):
+    """Specification for a callback function with optional arguments."""
+    func: Callable
+    args: tuple | None = None
+    kwargs: dict | None = None
+
+
+class RunConfig(BaseConfig):
+    inputs: dict | None = msgspec.field(default_factory=dict)
+    final_vars: list[str] | None = msgspec.field(default_factory=list)
+    config: dict | None = msgspec.field(default_factory=dict)
+    cache: dict | bool | None = msgspec.field(default=False)
+    with_adapter: WithAdapterConfig = msgspec.field(default_factory=WithAdapterConfig)
+    executor: ExecutorConfig = msgspec.field(default_factory=ExecutorConfig)
+    log_level: str | None = msgspec.field(default="INFO")
+    max_retries: int = msgspec.field(default=3)
+    retry_delay: int | float = msgspec.field(default=1)
+    jitter_factor: float | None = msgspec.field(default=0.1)
+    retry_exceptions: list[str] = msgspec.field(default_factory=lambda: ["Exception"])
+    # New fields for comprehensive configuration
+    pipeline_adapter_cfg: dict | None = msgspec.field(default=None)
+    project_adapter_cfg: dict | None = msgspec.field(default=None)
+    adapter: dict[str, Any] | None = msgspec.field(default=None)
+    reload: bool = msgspec.field(default=False)
+    on_success: CallbackSpec | None = msgspec.field(default=None)
+    on_failure: CallbackSpec | None = msgspec.field(default=None)
 
     def __post_init__(self):
         if isinstance(self.inputs, dict):
@@ -81,3 +107,30 @@ class RunConfig(BaseConfig):
                 else:
                     converted_exceptions.append(Exception)
             self.retry_exceptions = converted_exceptions
+
+        # Handle callback conversions
+        if self.on_success is not None and not isinstance(self.on_success, CallbackSpec):
+            if callable(self.on_success):
+                self.on_success = CallbackSpec(func=self.on_success)
+            elif isinstance(self.on_success, tuple) and len(self.on_success) == 3:
+                func, args, kwargs = self.on_success
+                self.on_success = CallbackSpec(func=func, args=args, kwargs=kwargs)
+            else:
+                self.on_success = None
+                warnings.warn(
+                    "Invalid on_success format, must be Callable or (Callable, args, kwargs)",
+                    RuntimeWarning
+                )
+
+        if self.on_failure is not None and not isinstance(self.on_failure, CallbackSpec):
+            if callable(self.on_failure):
+                self.on_failure = CallbackSpec(func=self.on_failure)
+            elif isinstance(self.on_failure, tuple) and len(self.on_failure) == 3:
+                func, args, kwargs = self.on_failure
+                self.on_failure = CallbackSpec(func=func, args=args, kwargs=kwargs)
+            else:
+                self.on_failure = None
+                warnings.warn(
+                    "Invalid on_failure format, must be Callable or (Callable, args, kwargs)",
+                    RuntimeWarning
+                )

{flowerpower-0.21.0 → flowerpower-0.30.0}/src/flowerpower/cli/__init__.py

@@ -19,13 +19,6 @@ app.add_typer(
 )
 
 
-if importlib.util.find_spec("paho"):
-    from .mqtt import app as mqtt_app
-
-    app.add_typer(
-        mqtt_app, name="mqtt", help="Connect pipelines to MQTT message brokers"
-    )
-
 
 @app.command()
 def init(

{flowerpower-0.21.0 → flowerpower-0.30.0}/src/flowerpower/pipeline/manager.py

@@ -40,13 +40,11 @@ class PipelineManager:
     - Configuration management and loading
     - Pipeline creation, deletion, and discovery
     - Pipeline execution via PipelineRunner
-    - Job scheduling via PipelineScheduler
     - Visualization via PipelineVisualizer
     - Import/export operations via PipelineIOManager
 
     Attributes:
         registry (PipelineRegistry): Handles pipeline registration and discovery
-        scheduler (PipelineScheduler): Manages job scheduling and execution
         visualizer (PipelineVisualizer): Handles pipeline visualization
         io (PipelineIOManager): Manages pipeline import/export operations
         project_cfg (ProjectConfig): Current project configuration

flowerpower-0.21.0/src/flowerpower/cli/mqtt.py

@@ -1,168 +0,0 @@
-import importlib
-import sys
-
-import typer
-
-from ..plugins.mqtt import run_pipeline_on_message as run_pipeline_on_message_
-from ..plugins.mqtt import start_listener as start_listener_
-from .utils import load_hook, parse_dict_or_list_param
-
-app = typer.Typer(help="MQTT management commands")
-
-
-@app.command()
-def start_listener(
-    on_message: str,
-    topic: str,
-    base_dir: str,
-    host: str = "localhost",
-    port: int = 1883,
-    username: str | None = None,
-    password: str | None = None,
-):
-    """Start an MQTT client to listen to messages on a topic
-
-    The connection to the MQTT broker is established using the provided configuration o a
-    MQTT event broker defined in the project configuration file `conf/project.yml`.
-    If not configuration is found, you have to provide the connection parameters,
-    such as `host`, `port`, `username`, and `password`.
-
-    The `on_message` module should contain a function `on_message` that will be called
-    with the message payload as argument.
-
-    Args:
-        on_message: Name of the module containing the on_message function
-        topic: MQTT topic to listen to
-        base_dir: Base directory for the module
-        host: MQTT broker host
-        port: MQTT broker port
-        username: MQTT broker username
-        password: MQTT broker password
-
-    Examples:
-        $ flowerpower mqtt start_listener --on-message my_module --topic my_topic --base-dir /path/to/module
-    """
-    sys.path.append(base_dir)
-    on_message_module = importlib.import_module(on_message)
-    start_listener_(
-        on_message=on_message_module.on_message,
-        topic=topic,
-        base_dir=base_dir,
-        host=host,
-        port=port,
-        username=username,
-        password=password,
-        background=False,
-    )
-
-
-@app.command()
-def run_pipeline_on_message(
-    name: str,
-    topic: str | None = None,
-    executor: str | None = None,
-    base_dir: str | None = None,
-    inputs: str | None = None,
-    final_vars: str | None = None,
-    config: str | None = None,
-    with_tracker: bool = False,
-    with_opentelemetry: bool = False,
-    with_progressbar: bool = False,
-    storage_options: str | None = None,
-    host: str | None = None,
-    port: int | None = None,
-    username: str | None = None,
-    password: str | None = None,
-    clean_session: bool = True,
-    qos: int = 0,
-    client_id: str | None = None,
-    client_id_suffix: str | None = None,
-    config_hook: str | None = None,
-    max_retries: int = typer.Option(
-        3, help="Maximum number of retry attempts if pipeline execution fails"
-    ),
-    retry_delay: float = typer.Option(
-        1.0, help="Base delay between retries in seconds"
-    ),
-    jitter_factor: float = typer.Option(
-        0.1, help="Random factor (0-1) applied to delay for jitter"
-    ),
-):
-    """Run a pipeline on a message
-
-    This command sets up an MQTT listener that executes a pipeline whenever a message is
-    received on the specified topic. The pipeline can be configured to retry on failure
-    using exponential backoff with jitter for better resilience.
-
-    Args:
-        name: Name of the pipeline
-        topic: MQTT topic to listen to
-        executor: Name of the executor
-        base_dir: Base directory for the pipeline
-        inputs: Inputs as JSON or key=value pairs or dict string
-        final_vars: Final variables as JSON or list
-        config: Config for the hamilton pipeline executor
-        with_tracker: Enable tracking with hamilton ui
-        with_opentelemetry: Enable OpenTelemetry tracing
-        with_progressbar: Enable progress bar
-        storage_options: Storage options as JSON, dict string or key=value pairs
-        host: MQTT broker host
-        port: MQTT broker port
-        username: MQTT broker username
-        password: MQTT broker password
-        clean_session: Whether to start a clean session with the broker
-        qos: MQTT Quality of Service level (0, 1, or 2)
-        client_id: Custom MQTT client identifier
-        client_id_suffix: Optional suffix to append to client_id
-        config_hook: Function to process incoming messages into pipeline config
-        max_retries: Maximum number of retry attempts if pipeline execution fails
-        retry_delay: Base delay between retries in seconds
-        jitter_factor: Random factor (0-1) applied to delay for jitter
-
-    Examples:
-        # Basic usage with a specific topic
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic sensors/data
-
-        # Configure retries for resilience
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic sensors/data --max-retries 5 --retry-delay 2.0
-
-        # Use a config hook to process messages
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic data/incoming --config-hook process_message
-
-
-    """
-
-    parsed_inputs = parse_dict_or_list_param(inputs, "dict")
-    parsed_config = parse_dict_or_list_param(config, "dict")
-    parsed_final_vars = parse_dict_or_list_param(final_vars, "list")
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict")
-
-    config_hook_function = None
-    if config_hook:
-        config_hook_function = load_hook(name, config_hook, base_dir, storage_options)
-
-    run_pipeline_on_message_(
-        name=name,
-        topic=topic,
-        executor=executor,
-        base_dir=base_dir,
-        inputs=parsed_inputs,
-        final_vars=parsed_final_vars,
-        config=parsed_config,
-        with_tracker=with_tracker,
-        with_opentelemetry=with_opentelemetry,
-        with_progressbar=with_progressbar,
-        storage_options=parsed_storage_options,
-        host=host,
-        port=port,
-        username=username,
-        password=password,
-        clean_session=clean_session,
-        qos=qos,
-        client_id=client_id,
-        client_id_suffix=client_id_suffix,
-        config_hook=config_hook_function,
-        max_retries=max_retries,
-        retry_delay=retry_delay,
-        jitter_factor=jitter_factor,
-    )

flowerpower-0.21.0/src/flowerpower/plugins/mqtt/__init__.py

@@ -1,8 +0,0 @@
-import warnings
-
-warnings.warn(
-    "The flowerpower.plugins.mqtt module is deprecated. "
-    "Please use 'flowerpower-mqtt' instead. Install it with 'pip install flowerpower-mqtt'.",
-    DeprecationWarning,
-    stacklevel=2,
-)