FlowerPower 0.20.0__py3-none-any.whl → 0.30.0__py3-none-any.whl

flowerpower/cli/mqtt.py

@@ -1,174 +0,0 @@
-import importlib
-import sys
-
-import typer
-
-from ..plugins.mqtt import run_pipeline_on_message as run_pipeline_on_message_
-from ..plugins.mqtt import start_listener as start_listener_
-from .utils import load_hook, parse_dict_or_list_param
-
-app = typer.Typer(help="MQTT management commands")
-
-
-@app.command()
-def start_listener(
-    on_message: str,
-    topic: str,
-    base_dir: str,
-    host: str = "localhost",
-    port: int = 1883,
-    username: str | None = None,
-    password: str | None = None,
-):
-    """Start an MQTT client to listen to messages on a topic
-
-    The connection to the MQTT broker is established using the provided configuration o a
-    MQTT event broker defined in the project configuration file `conf/project.yml`.
-    If not configuration is found, you have to provide the connection parameters,
-    such as `host`, `port`, `username`, and `password`.
-
-    The `on_message` module should contain a function `on_message` that will be called
-    with the message payload as argument.
-
-    Args:
-        on_message: Name of the module containing the on_message function
-        topic: MQTT topic to listen to
-        base_dir: Base directory for the module
-        host: MQTT broker host
-        port: MQTT broker port
-        username: MQTT broker username
-        password: MQTT broker password
-
-    Examples:
-        $ flowerpower mqtt start_listener --on-message my_module --topic my_topic --base-dir /path/to/module
-    """
-    sys.path.append(base_dir)
-    on_message_module = importlib.import_module(on_message)
-    start_listener_(
-        on_message=on_message_module.on_message,
-        topic=topic,
-        base_dir=base_dir,
-        host=host,
-        port=port,
-        username=username,
-        password=password,
-        background=False,
-    )
-
-
-@app.command()
-def run_pipeline_on_message(
-    name: str,
-    topic: str | None = None,
-    executor: str | None = None,
-    base_dir: str | None = None,
-    inputs: str | None = None,
-    final_vars: str | None = None,
-    config: str | None = None,
-    with_tracker: bool = False,
-    with_opentelemetry: bool = False,
-    with_progressbar: bool = False,
-    storage_options: str | None = None,
-    as_job: bool = False,
-    host: str | None = None,
-    port: int | None = None,
-    username: str | None = None,
-    password: str | None = None,
-    clean_session: bool = True,
-    qos: int = 0,
-    client_id: str | None = None,
-    client_id_suffix: str | None = None,
-    config_hook: str | None = None,
-    max_retries: int = typer.Option(
-        3, help="Maximum number of retry attempts if pipeline execution fails"
-    ),
-    retry_delay: float = typer.Option(
-        1.0, help="Base delay between retries in seconds"
-    ),
-    jitter_factor: float = typer.Option(
-        0.1, help="Random factor (0-1) applied to delay for jitter"
-    ),
-):
-    """Run a pipeline on a message
-
-    This command sets up an MQTT listener that executes a pipeline whenever a message is
-    received on the specified topic. The pipeline can be configured to retry on failure
-    using exponential backoff with jitter for better resilience.
-
-    Args:
-        name: Name of the pipeline
-        topic: MQTT topic to listen to
-        executor: Name of the executor
-        base_dir: Base directory for the pipeline
-        inputs: Inputs as JSON or key=value pairs or dict string
-        final_vars: Final variables as JSON or list
-        config: Config for the hamilton pipeline executor
-        with_tracker: Enable tracking with hamilton ui
-        with_opentelemetry: Enable OpenTelemetry tracing
-        with_progressbar: Enable progress bar
-        storage_options: Storage options as JSON, dict string or key=value pairs
-        as_job: Run as a job in the scheduler
-        host: MQTT broker host
-        port: MQTT broker port
-        username: MQTT broker username
-        password: MQTT broker password
-        clean_session: Whether to start a clean session with the broker
-        qos: MQTT Quality of Service level (0, 1, or 2)
-        client_id: Custom MQTT client identifier
-        client_id_suffix: Optional suffix to append to client_id
-        config_hook: Function to process incoming messages into pipeline config
-        max_retries: Maximum number of retry attempts if pipeline execution fails
-        retry_delay: Base delay between retries in seconds
-        jitter_factor: Random factor (0-1) applied to delay for jitter
-
-    Examples:
-        # Basic usage with a specific topic
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic sensors/data
-
-        # Configure retries for resilience
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic sensors/data --max-retries 5 --retry-delay 2.0
-
-        # Run as a job with custom MQTT settings
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic events/process --as-job --qos 2 --host mqtt.example.com
-
-        # Use a config hook to process messages
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic data/incoming --config-hook process_message
-
-
-    """
-
-    parsed_inputs = parse_dict_or_list_param(inputs, "dict")
-    parsed_config = parse_dict_or_list_param(config, "dict")
-    parsed_final_vars = parse_dict_or_list_param(final_vars, "list")
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict")
-
-    config_hook_function = None
-    if config_hook:
-        config_hook_function = load_hook(name, config_hook, base_dir, storage_options)
-
-    run_pipeline_on_message_(
-        name=name,
-        topic=topic,
-        executor=executor,
-        base_dir=base_dir,
-        inputs=parsed_inputs,
-        final_vars=parsed_final_vars,
-        config=parsed_config,
-        with_tracker=with_tracker,
-        with_opentelemetry=with_opentelemetry,
-        with_progressbar=with_progressbar,
-        storage_options=parsed_storage_options,
-        as_job=as_job,
-        host=host,
-        port=port,
-        username=username,
-        password=password,
-        clean_session=clean_session,
-        qos=qos,
-        client_id=client_id,
-        client_id_suffix=client_id_suffix,
-        config_hook=config_hook_function,
-        max_retries=max_retries,
-        retry_delay=retry_delay,
-        jitter_factor=jitter_factor,
-    )

flowerpower/job_queue/__init__.py

@@ -1,205 +0,0 @@
-import importlib
-from typing import Any, Optional
-
-from fsspec_utils import AbstractFileSystem
-from loguru import logger
-
-from ..cfg.project import ProjectConfig
-from ..utils.logging import setup_logging
-
-if importlib.util.find_spec("rq"):
-    from .rq import RQBackend, RQManager
-else:
-    RQBackend = None
-    RQManager = None
-from .base import BaseBackend, BaseJobQueueManager
-
-setup_logging()
-
-
-class JobQueueBackend:
-    """A factory class for creating backend instances for different job queue types.
-
-    This class provides a unified interface for creating backend instances that handle
-    the storage, queuing, and event management for different job queue types. Each backend
-    type provides specific implementations for:
-    - Job storage and persistence
-    - Queue management
-    - Event handling and communication
-    - Result storage
-
-    Example:
-        ```python
-        # Create RQ backend with Redis
-        rq_backend = JobQueueBackend(
-            job_queue_type="rq",
-            uri="redis://localhost:6379/0",
-            queues=["high", "default", "low"]
-        )
-
-        ```
-    """
-
-    def __new__(
-        cls,
-        job_queue_type: str,
-        **kwargs,
-    ) -> BaseBackend:
-        """Create a new backend instance based on the specified job queue type.
-
-        Args:
-            job_queue_type: The type of backend to create. Valid values are:
-                - "rq": Redis Queue backend using Redis
-            **kwargs: Backend-specific configuration options:
-                For RQ:
-                    - uri (str): Redis connection URI
-                    - queues (list[str]): List of queue names
-                    - result_ttl (int): Time to live for results in seconds
-
-        Returns:
-            BaseBackend: An instance of RQBackend depending on
-                the specified job queue type.
-
-        Raises:
-            ValueError: If an invalid job queue type is specified.
-            RuntimeError: If backend initialization fails due to configuration
-                or connection issues.
-
-        Example:
-            ```python
-            # Create RQ backend
-            rq_backend = Backend(
-                job_queue_type="rq",
-                uri="redis://localhost:6379/0",
-                queues=["high", "default", "low"],
-                result_ttl=3600
-            )
-
-            ```
-        """
-        if job_queue_type == "rq" and RQBackend is not None:
-            return RQBackend(**kwargs)
-        else:
-            if job_queue_type == "rq" and RQBackend is None:
-                logger.warning(
-                    "RQ is not installed. `JobQueueBackend` is not initialized and using the job queue is disabled. "
-                    "Install rq to use RQ. `uv pip install flowerpower[rq]` or `uv add flowerpower[rq]`"
-                )
-                return None
-            else:
-                raise ValueError(
-                    f"Invalid job queue type: {job_queue_type}. Valid types: ['rq']"
-                )
-
-
-class JobQueueManager:
-    """A factory class for creating job queue instances for job scheduling and execution.
-
-    This class provides a unified interface for creating different types of job queue instances
-    (RQ, APScheduler, Huey) based on the specified backend type. Each job queue type provides
-    different capabilities for job scheduling and execution.
-
-    The job queue instances handle:
-    - Job scheduling and execution
-    - Background task processing
-    - Job queue management
-    - Result storage and retrieval
-
-    Example:
-        ```python
-        # Create an RQ job queue
-        rq_worker = JobQueueManager(
-            type="rq",
-            name="my_worker",
-            log_level="DEBUG"
-        )
-
-
-        ```
-    """
-
-    def __new__(
-        cls,
-        type: str | None = None,
-        name: str | None = None,
-        base_dir: str | None = ".",
-        backend: JobQueueBackend | None = None,
-        storage_options: Optional[dict[str, Any]] = None,
-        fs: AbstractFileSystem | None = None,
-        log_level: str | None = None,
-        **kwargs,
-    ) -> BaseJobQueueManager:
-        """Create a new job queue instance based on the specified backend type.
-
-        Args:
-            type: The type of job queue to create. Valid values are:
-                - "rq": Redis Queue job queue for Redis-based job queuing
-            name: Name of the job queue instance. Used for identification in logs
-                and monitoring.
-            base_dir: Base directory for job queue files and configuration. Defaults
-                to current working directory if not specified.
-            backend: Pre-configured backend instance. If not provided, one will
-                be created based on configuration settings.
-            storage_options: Options for configuring filesystem storage access.
-                Example: {"mode": "async", "root": "/tmp", "protocol": "s3"}
-            fs: Custom filesystem implementation for storage operations.
-                Example: S3FileSystem, LocalFileSystem, etc.
-            log_level: Logging level for the job queue. Valid values are:
-                "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
-            **kwargs: Additional configuration options passed to the specific
-                job queue implementation.
-
-        Returns:
-            BaseJobQueueManager: An instance of the specified job queue type (RQManager).
-
-        Raises:
-            ValueError: If an invalid job queue type is specified.
-            ImportError: If required dependencies for the chosen job queue type
-                are not installed.
-            RuntimeError: If job queue initialization fails due to configuration
-                or connection issues.
-
-        Example:
-            ```python
-            # Basic RQ job queue
-            worker = JobQueueManager(type="rq", name="basic_worker")
-
-
-            ```
-        """
-        if type is None:
-            type = ProjectConfig.load(
-                base_dir=base_dir,
-                name=name,
-                fs=fs,
-                storage_options=storage_options or {},
-            ).job_queue.type
-
-        if type == "rq":
-            if RQManager is not None:
-                return RQManager(
-                    name=name,
-                    base_dir=base_dir,
-                    backend=backend,
-                    storage_options=storage_options,
-                    fs=fs,
-                    log_level=log_level,
-                    **kwargs,
-                )
-            else:
-                logger.warning(
-                    "`JobQueueManager` can not be initialized. This might be due to missing dependencies (RQ), invalid configuration or backend not being available."
-                )
-                return None
-
-        else:
-            raise ImportError(f"Invalid job queue type: {type}. Valid types: ['rq']")
-
-
-__all__ = [
-    "JobQueueManager",
-    "RQManager",
-    # "HueyWorker",
-    "JobQueueBackend",
-    "RQBackend",
-]