FlowerPower 0.11.6.19__py3-none-any.whl → 0.20.0__py3-none-any.whl

flowerpower/job_queue/__init__.py

@@ -1,17 +1,12 @@
 import importlib
 from typing import Any, Optional
 
+from fsspec_utils import AbstractFileSystem
 from loguru import logger
 
 from ..cfg.project import ProjectConfig
-from ..fs import AbstractFileSystem
 from ..utils.logging import setup_logging
 
-if importlib.util.find_spec("apscheduler"):
-    from .apscheduler import APSBackend, APSManager
-else:
-    APSBackend = None
-    APSManager = None
 if importlib.util.find_spec("rq"):
     from .rq import RQBackend, RQManager
 else:
@@ -42,18 +37,6 @@ class JobQueueBackend:
             queues=["high", "default", "low"]
         )
 
-        # Create APScheduler backend with PostgreSQL and Redis
-        aps_backend = JobQueueBackend(
-            job_queue_type="apscheduler",
-            data_store={
-                "type": "postgresql",
-                "uri": "postgresql+asyncpg://user:pass@localhost/db"
-            },
-            event_broker={
-                "type": "redis",
-                "uri": "redis://localhost:6379/0"
-            }
-        )
         ```
     """
 
@@ -67,21 +50,14 @@ class JobQueueBackend:
         Args:
             job_queue_type: The type of backend to create. Valid values are:
                 - "rq": Redis Queue backend using Redis
-                - "apscheduler": APScheduler backend supporting various databases
-                    and event brokers
             **kwargs: Backend-specific configuration options:
                 For RQ:
                     - uri (str): Redis connection URI
                     - queues (list[str]): List of queue names
                     - result_ttl (int): Time to live for results in seconds
-                For APScheduler:
-                    - data_store (dict): Data store configuration
-                    - event_broker (dict): Event broker configuration
-                    - cleanup_interval (int): Cleanup interval in seconds
-                    - max_concurrent_jobs (int): Maximum concurrent jobs
 
         Returns:
-            BaseBackend: An instance of RQBackend or APSBackend depending on
+            BaseBackend: An instance of RQBackend depending on
                 the specified job queue type.
 
         Raises:
@@ -99,27 +75,10 @@ class JobQueueBackend:
                 result_ttl=3600
             )
 
-            # Create APScheduler backend with PostgreSQL and Redis
-            aps_backend = Backend(
-                job_queue_type="apscheduler",
-                data_store={
-                    "type": "postgresql",
-                    "uri": "postgresql+asyncpg://user:pass@localhost/db",
-                    "schema": "scheduler"
-                },
-                event_broker={
-                    "type": "redis",
-                    "uri": "redis://localhost:6379/0"
-                },
-                cleanup_interval=300,
-                max_concurrent_jobs=10
-            )
             ```
         """
         if job_queue_type == "rq" and RQBackend is not None:
             return RQBackend(**kwargs)
-        elif job_queue_type == "apscheduler" and APSBackend is not None:
-            return APSBackend(**kwargs)
         else:
             if job_queue_type == "rq" and RQBackend is None:
                 logger.warning(
@@ -127,15 +86,9 @@ class JobQueueBackend:
                     "Install rq to use RQ. `uv pip install flowerpower[rq]` or `uv add flowerpower[rq]`"
                 )
                 return None
-            elif job_queue_type == "apscheduler" and APSBackend is None:
-                logger.warning(
-                    "APScheduler is not installed. `JobQueueBackend` is not initialized and using the job queue is disabled. "
-                    "Install apscheduler to use APScheduler. `uv pip install flowerpower[apscheduler]` or `uv add flowerpower[apscheduler]`"
-                )
-                return None
             else:
                 raise ValueError(
-                    f"Invalid job queue type: {job_queue_type}. Valid types: ['rq', 'apscheduler']"
+                    f"Invalid job queue type: {job_queue_type}. Valid types: ['rq']"
                 )
 
 
@@ -161,17 +114,6 @@ class JobQueueManager:
             log_level="DEBUG"
         )
 
-        # Create an APScheduler job queue with custom backend
-        from flowerpower.job_queue.apscheduler import APSBackend
-        backend_config = APSBackend(
-            data_store={"type": "postgresql", "uri": "postgresql+asyncpg://user:pass@localhost/db"},
-            event_broker={"type": "redis", "uri": "redis://localhost:6379/0"}
-        )
-        aps_worker = JobQueueManager(
-            type="apscheduler",
-            name="scheduler",
-            backend=backend_config
-        )
 
         ```
     """
@@ -192,7 +134,6 @@ class JobQueueManager:
         Args:
             type: The type of job queue to create. Valid values are:
                 - "rq": Redis Queue job queue for Redis-based job queuing
-                - "apscheduler": APScheduler job queue for advanced job scheduling
             name: Name of the job queue instance. Used for identification in logs
                 and monitoring.
             base_dir: Base directory for job queue files and configuration. Defaults
@@ -209,8 +150,7 @@ class JobQueueManager:
                 job queue implementation.
 
         Returns:
-            BaseJobQueueManager: An instance of the specified job queue type (RQManager,
-                APSManager).
+            BaseJobQueueManager: An instance of the specified job queue type (RQManager).
 
         Raises:
             ValueError: If an invalid job queue type is specified.
@@ -224,14 +164,6 @@ class JobQueueManager:
             # Basic RQ job queue
             worker = JobQueueManager(type="rq", name="basic_worker")
 
-            # APScheduler with custom logging and storage
-            worker = JobQueueManager(
-                type="apscheduler",
-                name="scheduler",
-                base_dir="/app/data",
-                storage_options={"mode": "async"},
-                log_level="DEBUG"
-            )
 
             ```
         """
@@ -260,35 +192,14 @@ class JobQueueManager:
                 )
                 return None
 
-        elif type == "apscheduler":
-            if APSManager is not None:
-                return APSManager(
-                    name=name,
-                    base_dir=base_dir,
-                    backend=backend,
-                    storage_options=storage_options,
-                    fs=fs,
-                    log_level=log_level,
-                    **kwargs,
-                )
-            else:
-                logger.warning(
-                    "`JobQueueManager` can not be initialized. This might be due to missing dependencies (APScheduler), invalid configuration or backend not being available."
-                )
-                return None
-
         else:
-            raise ImportError(
-                f"Invalid job queue type: {type}. Valid types: ['rq', 'apscheduler']"
-            )
+            raise ImportError(f"Invalid job queue type: {type}. Valid types: ['rq']")
 
 
 __all__ = [
     "JobQueueManager",
     "RQManager",
-    "APSManager",
     # "HueyWorker",
     "JobQueueBackend",
     "RQBackend",
-    "APSBackend",
 ]

flowerpower/job_queue/base.py

@@ -14,7 +14,9 @@ import urllib.parse
 from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
-from typing import Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from loguru import logger
 
 if importlib.util.find_spec("sqlalchemy"):
     from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
@@ -22,8 +24,13 @@ else:
     create_async_engine = None
     AsyncEngine = TypeVar("AsyncEngine")
 
+# Import PipelineRegistry with TYPE_CHECKING to avoid circular imports
+if TYPE_CHECKING:
+    from ..pipeline.registry import PipelineRegistry
+
+from fsspec_utils import AbstractFileSystem, filesystem
+
 from ..cfg import ProjectConfig
-from ..fs import AbstractFileSystem, get_filesystem
 # from ..utils.misc import update_config_from_dict
 from ..settings import BACKEND_PROPERTIES, CACHE_DIR, CONFIG_DIR, PIPELINES_DIR
 
@@ -357,6 +364,9 @@ class BaseJobQueueManager:
         self._pipelines_dir = kwargs.get("pipelines_dir", PIPELINES_DIR)
         self._cfg_dir = CONFIG_DIR
 
+        # Initialize pipeline registry (will be injected by FlowerPowerProject)
+        self._pipeline_registry = None
+
         if storage_options is not None:
             cached = True
             cache_storage = posixpath.join(
@@ -367,7 +377,7 @@ class BaseJobQueueManager:
             cached = False
             cache_storage = None
         if not fs:
-            fs = get_filesystem(
+            fs = filesystem(
                 self._base_dir,
                 storage_options=storage_options,
                 cached=cached,
@@ -411,3 +421,191 @@ class BaseJobQueueManager:
 
         if modules_path not in sys.path:
             sys.path.insert(0, modules_path)
+
+    @property
+    def pipeline_registry(self) -> "PipelineRegistry":
+        """Get or create a PipelineRegistry instance for this job queue manager.
+
+        This property lazily creates a PipelineRegistry using the job queue manager's
+        filesystem and directory configuration. The registry is cached after first access.
+
+        Returns:
+            PipelineRegistry: A registry instance configured with this manager's settings
+
+        Raises:
+            RuntimeError: If PipelineRegistry creation fails
+
+        Example:
+            ```python
+            manager = RQManager(base_dir="/path/to/project")
+            registry = manager.pipeline_registry  # Creates registry on first access
+            pipeline = registry.get_pipeline("my_pipeline")
+            ```
+        """
+        if self._pipeline_registry is None:
+            try:
+                # Import here to avoid circular import issues
+                from ..pipeline.registry import PipelineRegistry
+
+                # Create registry using the from_filesystem factory method
+                self._pipeline_registry = PipelineRegistry.from_filesystem(
+                    base_dir=self._base_dir,
+                    fs=self._fs,
+                    storage_options=self._storage_options,
+                )
+
+                logger.debug(
+                    f"Created PipelineRegistry for JobQueueManager with base_dir: {self._base_dir}"
+                )
+
+            except Exception as e:
+                error_msg = f"Failed to create PipelineRegistry: {e}"
+                logger.error(error_msg)
+                raise RuntimeError(error_msg) from e
+
+        return self._pipeline_registry
+
+    # --- Pipeline-specific high-level methods ---
+
+    def schedule_pipeline(self, name: str, *args, **kwargs):
+        """Schedule a pipeline for execution using its name.
+
+        This high-level method loads the pipeline from the internal registry and schedules
+        its execution with the job queue.
+
+        Args:
+            name: Name of the pipeline to schedule
+            *args: Additional positional arguments for scheduling
+            **kwargs: Additional keyword arguments for scheduling
+
+        Returns:
+            Schedule ID or job ID depending on implementation
+
+        Raises:
+            NotImplementedError: Must be implemented by subclasses
+        """
+        raise NotImplementedError("Subclasses must implement schedule_pipeline()")
+
+    def enqueue_pipeline(self, name: str, *args, **kwargs):
+        """Enqueue a pipeline for immediate execution using its name.
+
+        This high-level method loads the pipeline from the internal registry and enqueues
+        it for immediate execution in the job queue.
+
+        Args:
+            name: Name of the pipeline to enqueue
+            *args: Additional positional arguments for job execution
+            **kwargs: Additional keyword arguments for job execution
+
+        Returns:
+            Job ID or result depending on implementation
+
+        Raises:
+            NotImplementedError: Must be implemented by subclasses
+        """
+        raise NotImplementedError("Subclasses must implement enqueue_pipeline()")
+
+    # --- Core job queue methods ---
+
+    def enqueue(self, func, *args, **kwargs):
+        """Enqueue a job for execution (immediate, delayed, or scheduled).
+
+        This is the main method for adding jobs to the queue. It supports:
+        - Immediate execution (no run_at or run_in parameters)
+        - Delayed execution (run_in parameter)
+        - Scheduled execution (run_at parameter)
+
+        Args:
+            func: Function to execute. Must be importable from the worker process.
+            *args: Positional arguments for the function
+            **kwargs: Keyword arguments including:
+                - run_in: Schedule the job to run after a delay (timedelta, int seconds, or string)
+                - run_at: Schedule the job to run at a specific datetime
+                - Other job queue specific parameters (timeout, retry, etc.)
+
+        Returns:
+            Job object or job ID depending on implementation
+
+        Raises:
+            NotImplementedError: Must be implemented by subclasses
+
+        Example:
+            ```python
+            # Immediate execution
+            manager.enqueue(my_func, arg1, arg2, kwarg1="value")
+
+            # Delayed execution
+            manager.enqueue(my_func, arg1, run_in=300)  # 5 minutes
+            manager.enqueue(my_func, arg1, run_in=timedelta(hours=1))
+
+            # Scheduled execution
+            manager.enqueue(my_func, arg1, run_at=datetime(2025, 1, 1, 9, 0))
+            ```
+        """
+        raise NotImplementedError("Subclasses must implement enqueue()")
+
+    def enqueue_in(self, delay, func, *args, **kwargs):
+        """Enqueue a job to run after a specified delay.
+
+        This is a convenience method for delayed execution. It's equivalent to
+        calling enqueue() with the run_in parameter.
+
+        Args:
+            delay: Time to wait before execution (timedelta, int seconds, or string)
+            func: Function to execute
+            *args: Positional arguments for the function
+            **kwargs: Keyword arguments for the function and job options
+
+        Returns:
+            Job object or job ID depending on implementation
+
+        Raises:
+            NotImplementedError: Must be implemented by subclasses
+
+        Example:
+            ```python
+            # Run in 5 minutes
+            manager.enqueue_in(300, my_func, arg1, arg2)
+
+            # Run in 1 hour
+            manager.enqueue_in(timedelta(hours=1), my_func, arg1, kwarg1="value")
+
+            # Run in 30 seconds (string format)
+            manager.enqueue_in("30s", my_func, arg1)
+            ```
+        """
+        raise NotImplementedError("Subclasses must implement enqueue_in()")
+
+    def enqueue_at(self, datetime, func, *args, **kwargs):
+        """Enqueue a job to run at a specific datetime.
+
+        This is a convenience method for scheduled execution. It's equivalent to
+        calling enqueue() with the run_at parameter.
+
+        Args:
+            datetime: When to execute the job (datetime object or ISO string)
+            func: Function to execute
+            *args: Positional arguments for the function
+            **kwargs: Keyword arguments for the function and job options
+
+        Returns:
+            Job object or job ID depending on implementation
+
+        Raises:
+            NotImplementedError: Must be implemented by subclasses
+
+        Example:
+            ```python
+            # Run at specific time
+            manager.enqueue_at(datetime(2025, 1, 1, 9, 0), my_func, arg1, arg2)
+
+            # Run tomorrow at 9 AM
+            tomorrow_9am = datetime.now() + timedelta(days=1)
+            tomorrow_9am = tomorrow_9am.replace(hour=9, minute=0, second=0)
+            manager.enqueue_at(tomorrow_9am, my_func, arg1, kwarg1="value")
+
+            # Run using ISO string
+            manager.enqueue_at("2025-01-01T09:00:00", my_func, arg1)
+            ```
+        """
+        raise NotImplementedError("Subclasses must implement enqueue_at()")

flowerpower/job_queue/rq/concurrent_workers/thread_worker.py

@@ -1,11 +1,8 @@
 # filepath: /Volumes/WD_Blue_1TB/coding/libs/flowerpower/src/flowerpower/worker/rq/concurrent_workers.py
 import concurrent.futures
 import datetime as dt
-import logging
 import os
-import threading
 import time
-import traceback
 from concurrent.futures import ThreadPoolExecutor
 
 from loguru import logger