FlowerPower 0.9.13.1__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

flowerpower/job_queue/rq/setup.py

@@ -0,0 +1,150 @@
+from dataclasses import dataclass, field
+
+import redis
+
+from ..base import BaseBackend
+
+# Enums for RQ DataStore and EventBroker types
+# class RQBackendType(BackendType):
+#    REDIS = "redis"
+#    MEMORY = "memory"
+
+
+@dataclass  # (slots=True)
+class RQBackend(BaseBackend):
+    """RQ Backend implementation for Redis Queue (RQ) job storage and queuing.
+
+    This class provides a Redis-based backend for RQ job storage and queue management.
+    It supports both Redis and in-memory storage options for development/testing.
+
+    Args:
+        queues (str | list[str] | None): Names of queues to create. Defaults to ["default"].
+        num_workers (int): Number of worker processes to use. Defaults to 1.
+
+    Attributes:
+        type (str): Backend type, either "redis" or "memory". Inherited from BaseBackend.
+        uri (str): Connection URI. Inherited from BaseBackend.
+        result_namespace (str): Namespace for storing job results in Redis.
+        _client (redis.Redis | dict): Redis client or dict for memory storage.
+
+    Raises:
+        ValueError: If an invalid backend type is specified.
+
+    Example:
+        ```python
+        # Create Redis backend with default queue
+        backend = RQBackend(
+            type="redis",
+            uri="redis://localhost:6379/0"
+        )
+
+        # Create Redis backend with multiple queues
+        backend = RQBackend(
+            type="redis",
+            uri="redis://localhost:6379/0",
+            queues=["high", "default", "low"]
+        )
+
+        # Create in-memory backend for testing
+        backend = RQBackend(type="memory", queues=["test"])
+        ```
+    """
+
+    queues: str | list[str] | None = field(default_factory=lambda: ["default"])
+    num_workers: int = field(default=1)
+
+    def __post_init__(self) -> None:
+        """Initialize and validate the backend configuration.
+
+        This method is called automatically after instance creation. It:
+        1. Sets default type to "redis" if not specified
+        2. Calls parent class initialization
+        3. Validates backend type
+        4. Sets default result namespace
+
+        Raises:
+            ValueError: If an unsupported backend type is specified.
+                Only "redis" and "memory" types are supported.
+        """
+        if self.type is None:
+            self.type = "redis"
+        super().__post_init__()
+
+        if not self.type.is_memory_type and not self.type.is_redis_type:
+            raise ValueError(
+                f"Invalid backend type: {self.type}. Valid types: {[self.type.REDIS, self.type.MEMORY]}"
+            )
+
+        self.result_namespace = getattr(self, "result_namespace", "flowerpower:results")
+
+    def setup(self) -> None:
+        """Set up the Redis client or in-memory storage.
+
+        This method initializes the backend storage based on the configured type.
+        For Redis, it creates a Redis client with the specified connection parameters.
+        For in-memory storage, it creates a simple dictionary.
+
+        Raises:
+            ValueError: If an unsupported backend type is specified.
+            redis.RedisError: If Redis connection fails.
+
+        Example:
+            ```python
+            backend = RQBackend(
+                type="redis",
+                host="localhost",
+                port=6379,
+                password="secret",
+                database="0",
+                ssl=True
+            )
+            backend.setup()
+            ```
+        """
+        # Use connection info from BaseBackend to create Redis client
+        if self.type.is_redis_type:
+            # Parse db from database or default to 0
+            db = 0
+            if self.database is not None:
+                try:
+                    db = int(self.database)
+                except Exception:
+                    db = 0
+            self._client = redis.Redis(
+                host=self.host or self.type.default_host,
+                port=self.port or self.type.default_port,
+                db=db,
+                password=self.password,
+                ssl=self.ssl,
+            )
+        elif self.type.is_memory_type:
+            # Simple in-memory dict for testing
+            self._client = {}
+        else:
+            raise ValueError(f"Unsupported RQBackend type: {self.type}")
+
+    @property
+    def client(self) -> redis.Redis | dict:
+        """Get the initialized storage client.
+
+        This property provides access to the Redis client or in-memory dictionary,
+            initializing it if needed.
+
+        Returns:
+            redis.Redis | dict: The Redis client for Redis backend,
+                or dictionary for in-memory backend.
+
+        Example:
+            ```python
+            backend = RQBackend(type="redis", uri="redis://localhost:6379/0")
+            redis_client = backend.client  # Gets Redis client
+            redis_client.set("key", "value")
+
+            backend = RQBackend(type="memory")
+            mem_dict = backend.client  # Gets dict for testing
+            mem_dict["key"] = "value"
+            ```
+        """
+        if self._client is None:
+            self.setup()
+        return self._client

flowerpower/job_queue/rq/utils.py

@@ -0,0 +1,69 @@
+from rich.console import Console
+from rich.table import Table
+from rq import Queue
+from rq_scheduler import Scheduler
+
+
+def show_schedules(scheduler: Scheduler) -> None:
+    """
+    Display the schedules in a user-friendly format.
+
+    Args:
+        scheduler (Scheduler): An instance of rq_scheduler.Scheduler.
+    """
+    console = Console()
+    table = Table(title="Scheduled Jobs")
+
+    table.add_column("ID", style="cyan")
+    table.add_column("Function", style="green")
+    table.add_column("Schedule", style="yellow")
+    table.add_column("Next Run", style="magenta")
+
+    for job in scheduler.get_jobs():
+        # Determine schedule type and format
+        schedule_type = "Unknown"
+        if hasattr(job, "meta"):
+            if job.meta.get("cron"):
+                schedule_type = f"Cron: {job.meta['cron']}"
+            elif job.meta.get("interval"):
+                schedule_type = f"Interval: {job.meta['interval']}s"
+
+        next_run = (
+            job.scheduled_at.strftime("%Y-%m-%d %H:%M:%S")
+            if hasattr(job, "scheduled_at") and job.scheduled_at
+            else "Unknown"
+        )
+
+        table.add_row(job.id, job.func_name, schedule_type, next_run)
+
+    console.print(table)
+
+
+def show_jobs(queue: Queue) -> None:
+    """
+    Display the jobs in a user-friendly format.
+
+    Args:
+        queue (Queue): An instance of rq.Queue.
+    """
+    console = Console()
+    table = Table(title="Jobs")
+
+    table.add_column("ID", style="cyan")
+    table.add_column("Function", style="green")
+    table.add_column("Status", style="yellow")
+    table.add_column("Enqueued At", style="magenta")
+    table.add_column("Result", style="blue")
+
+    for job in queue.get_jobs():
+        table.add_row(
+            job.id,
+            job.func_name,
+            job.get_status(),
+            job.enqueued_at.strftime("%Y-%m-%d %H:%M:%S")
+            if job.enqueued_at
+            else "Unknown",
+            str(job.result) if job.result else "None",
+        )
+
+    console.print(table)

flowerpower/pipeline/__init__.py

@@ -0,0 +1,5 @@
+from .manager import PipelineManager
+
+__all__ = [
+    "PipelineManager",
+]

flowerpower/pipeline/base.py

@@ -0,0 +1,118 @@
+import importlib
+import posixpath
+import sys
+from types import TracebackType
+
+from loguru import logger
+from munch import Munch
+
+from ..cfg import PipelineConfig, ProjectConfig
+from ..fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
+from ..utils.logging import setup_logging
+
+setup_logging()
+
+
+def load_module(name: str, reload: bool = False):
+    """
+    Load a module.
+
+    Args:
+        name (str): The name of the module.
+
+    Returns:
+        module: The loaded module.
+    """
+    if name in sys.modules:
+        if reload:
+            return importlib.reload(sys.modules[name])
+        return sys.modules[name]
+    return importlib.import_module(name)
+
+
+class BasePipeline:
+    """
+    Base class for all pipelines.
+    """
+
+    def __init__(
+        self,
+        base_dir: str | None = None,
+        storage_options: dict | Munch | BaseStorageOptions = {},
+        fs: AbstractFileSystem | None = None,
+        cfg_dir: str = "conf",
+        pipelines_dir: str = "pipelines",
+        job_queue_type: str | None = None,  # New parameter for worker backend
+    ):
+        self._base_dir = base_dir
+        self._storage_options = storage_options
+        if fs is None:
+            fs = get_filesystem(self._base_dir, **self._storage_options)
+        self._fs = fs
+        self._cfg_dir = cfg_dir
+        self._pipelines_dir = pipelines_dir
+        self._job_queue_type = job_queue_type
+
+        try:
+            self._fs.makedirs(f"{self._cfg_dir}/pipelines", exist_ok=True)
+            self._fs.makedirs(self._pipelines_dir, exist_ok=True)
+        except Exception as e:
+            logger.error(f"Error creating directories: {e}")
+
+        self._add_modules_path()
+
+    def __enter__(self) -> "BasePipeline":
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        pass
+
+    def _add_modules_path(self):
+        """
+        Sync the filesystem.
+
+        Returns:
+            None
+        """
+        if self._fs.is_cache_fs:
+            self._fs.sync()
+
+        modules_path = posixpath.join(self._fs.path, self._pipelines_dir)
+        if modules_path not in sys.path:
+            sys.path.insert(0, modules_path)
+
+    def _load_project_cfg(self) -> ProjectConfig:
+        """
+        Load the project configuration.
+
+        Returns:
+            ProjectConfig: The loaded project configuration.
+        """
+        return ProjectConfig.load(
+            base_dir=self._base_dir,
+            job_queue_type=self._job_queue_type,
+            fs=self._fs,
+            storage_options=self._storage_options,
+        )
+
+    def _load_pipeline_cfg(self, name: str) -> PipelineConfig:
+        """
+        Load the pipeline configuration.
+
+        Args:
+            name (str): The name of the pipeline.
+
+        Returns:
+            PipelineConfig: The loaded pipeline configuration.
+        """
+        return PipelineConfig.load(
+            base_dir=self._base_dir,
+            name=name,
+            fs=self._fs,
+            storage_options=self._storage_options,
+        )