FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl

flowerpower/job_queue/__init__.py

@@ -1,294 +0,0 @@
-import importlib
-from typing import Any, Optional
-
-from loguru import logger
-
-from ..cfg.project import ProjectConfig
-from ..fs import AbstractFileSystem
-from ..utils.logging import setup_logging
-
-if importlib.util.find_spec("apscheduler"):
-    from .apscheduler import APSBackend, APSManager
-else:
-    APSBackend = None
-    APSManager = None
-if importlib.util.find_spec("rq"):
-    from .rq import RQBackend, RQManager
-else:
-    RQBackend = None
-    RQManager = None
-from .base import BaseBackend, BaseJobQueueManager
-
-setup_logging()
-
-
-class JobQueueBackend:
-    """A factory class for creating backend instances for different job queue types.
-
-    This class provides a unified interface for creating backend instances that handle
-    the storage, queuing, and event management for different job queue types. Each backend
-    type provides specific implementations for:
-    - Job storage and persistence
-    - Queue management
-    - Event handling and communication
-    - Result storage
-
-    Example:
-        ```python
-        # Create RQ backend with Redis
-        rq_backend = JobQueueBackend(
-            job_queue_type="rq",
-            uri="redis://localhost:6379/0",
-            queues=["high", "default", "low"]
-        )
-
-        # Create APScheduler backend with PostgreSQL and Redis
-        aps_backend = JobQueueBackend(
-            job_queue_type="apscheduler",
-            data_store={
-                "type": "postgresql",
-                "uri": "postgresql+asyncpg://user:pass@localhost/db"
-            },
-            event_broker={
-                "type": "redis",
-                "uri": "redis://localhost:6379/0"
-            }
-        )
-        ```
-    """
-
-    def __new__(
-        cls,
-        job_queue_type: str,
-        **kwargs,
-    ) -> BaseBackend:
-        """Create a new backend instance based on the specified job queue type.
-
-        Args:
-            job_queue_type: The type of backend to create. Valid values are:
-                - "rq": Redis Queue backend using Redis
-                - "apscheduler": APScheduler backend supporting various databases
-                    and event brokers
-            **kwargs: Backend-specific configuration options:
-                For RQ:
-                    - uri (str): Redis connection URI
-                    - queues (list[str]): List of queue names
-                    - result_ttl (int): Time to live for results in seconds
-                For APScheduler:
-                    - data_store (dict): Data store configuration
-                    - event_broker (dict): Event broker configuration
-                    - cleanup_interval (int): Cleanup interval in seconds
-                    - max_concurrent_jobs (int): Maximum concurrent jobs
-
-        Returns:
-            BaseBackend: An instance of RQBackend or APSBackend depending on
-                the specified job queue type.
-
-        Raises:
-            ValueError: If an invalid job queue type is specified.
-            RuntimeError: If backend initialization fails due to configuration
-                or connection issues.
-
-        Example:
-            ```python
-            # Create RQ backend
-            rq_backend = Backend(
-                job_queue_type="rq",
-                uri="redis://localhost:6379/0",
-                queues=["high", "default", "low"],
-                result_ttl=3600
-            )
-
-            # Create APScheduler backend with PostgreSQL and Redis
-            aps_backend = Backend(
-                job_queue_type="apscheduler",
-                data_store={
-                    "type": "postgresql",
-                    "uri": "postgresql+asyncpg://user:pass@localhost/db",
-                    "schema": "scheduler"
-                },
-                event_broker={
-                    "type": "redis",
-                    "uri": "redis://localhost:6379/0"
-                },
-                cleanup_interval=300,
-                max_concurrent_jobs=10
-            )
-            ```
-        """
-        if job_queue_type == "rq" and RQBackend is not None:
-            return RQBackend(**kwargs)
-        elif job_queue_type == "apscheduler" and APSBackend is not None:
-            return APSBackend(**kwargs)
-        else:
-            if job_queue_type == "rq" and RQBackend is None:
-                logger.warning(
-                    "RQ is not installed. `JobQueueBackend` is not initialized and using the job queue is disabled. "
-                    "Install rq to use RQ. `uv pip install flowerpower[rq]` or `uv add flowerpower[rq]`"
-                )
-                return None
-            elif job_queue_type == "apscheduler" and APSBackend is None:
-                logger.warning(
-                    "APScheduler is not installed. `JobQueueBackend` is not initialized and using the job queue is disabled. "
-                    "Install apscheduler to use APScheduler. `uv pip install flowerpower[apscheduler]` or `uv add flowerpower[apscheduler]`"
-                )
-                return None
-            else:
-                raise ValueError(
-                    f"Invalid job queue type: {job_queue_type}. Valid types: ['rq', 'apscheduler']"
-                )
-
-
-class JobQueueManager:
-    """A factory class for creating job queue instances for job scheduling and execution.
-
-    This class provides a unified interface for creating different types of job queue instances
-    (RQ, APScheduler, Huey) based on the specified backend type. Each job queue type provides
-    different capabilities for job scheduling and execution.
-
-    The job queue instances handle:
-    - Job scheduling and execution
-    - Background task processing
-    - Job queue management
-    - Result storage and retrieval
-
-    Example:
-        ```python
-        # Create an RQ job queue
-        rq_worker = JobQueueManager(
-            type="rq",
-            name="my_worker",
-            log_level="DEBUG"
-        )
-
-        # Create an APScheduler job queue with custom backend
-        from flowerpower.job_queue.apscheduler import APSBackend
-        backend_config = APSBackend(
-            data_store={"type": "postgresql", "uri": "postgresql+asyncpg://user:pass@localhost/db"},
-            event_broker={"type": "redis", "uri": "redis://localhost:6379/0"}
-        )
-        aps_worker = JobQueueManager(
-            type="apscheduler",
-            name="scheduler",
-            backend=backend_config
-        )
-
-        ```
-    """
-
-    def __new__(
-        cls,
-        type: str | None = None,
-        name: str | None = None,
-        base_dir: str | None = ".",
-        backend: JobQueueBackend | None = None,
-        storage_options: Optional[dict[str, Any]] = None,
-        fs: AbstractFileSystem | None = None,
-        log_level: str | None = None,
-        **kwargs,
-    ) -> BaseJobQueueManager:
-        """Create a new job queue instance based on the specified backend type.
-
-        Args:
-            type: The type of job queue to create. Valid values are:
-                - "rq": Redis Queue job queue for Redis-based job queuing
-                - "apscheduler": APScheduler job queue for advanced job scheduling
-            name: Name of the job queue instance. Used for identification in logs
-                and monitoring.
-            base_dir: Base directory for job queue files and configuration. Defaults
-                to current working directory if not specified.
-            backend: Pre-configured backend instance. If not provided, one will
-                be created based on configuration settings.
-            storage_options: Options for configuring filesystem storage access.
-                Example: {"mode": "async", "root": "/tmp", "protocol": "s3"}
-            fs: Custom filesystem implementation for storage operations.
-                Example: S3FileSystem, LocalFileSystem, etc.
-            log_level: Logging level for the job queue. Valid values are:
-                "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
-            **kwargs: Additional configuration options passed to the specific
-                job queue implementation.
-
-        Returns:
-            BaseJobQueueManager: An instance of the specified job queue type (RQManager,
-                APSManager).
-
-        Raises:
-            ValueError: If an invalid job queue type is specified.
-            ImportError: If required dependencies for the chosen job queue type
-                are not installed.
-            RuntimeError: If job queue initialization fails due to configuration
-                or connection issues.
-
-        Example:
-            ```python
-            # Basic RQ job queue
-            worker = JobQueueManager(type="rq", name="basic_worker")
-
-            # APScheduler with custom logging and storage
-            worker = JobQueueManager(
-                type="apscheduler",
-                name="scheduler",
-                base_dir="/app/data",
-                storage_options={"mode": "async"},
-                log_level="DEBUG"
-            )
-
-            ```
-        """
-        if type is None:
-            type = ProjectConfig.load(
-                base_dir=base_dir,
-                name=name,
-                fs=fs,
-                storage_options=storage_options or {},
-            ).job_queue.type
-
-        if type == "rq":
-            if RQManager is not None:
-                return RQManager(
-                    name=name,
-                    base_dir=base_dir,
-                    backend=backend,
-                    storage_options=storage_options,
-                    fs=fs,
-                    log_level=log_level,
-                    **kwargs,
-                )
-            else:
-                logger.warning(
-                    "`JobQueueManager` can not be initialized. This might be due to missing dependencies (RQ), invalid configuration or backend not being available."
-                )
-                return None
-
-        elif type == "apscheduler":
-            if APSManager is not None:
-                return APSManager(
-                    name=name,
-                    base_dir=base_dir,
-                    backend=backend,
-                    storage_options=storage_options,
-                    fs=fs,
-                    log_level=log_level,
-                    **kwargs,
-                )
-            else:
-                logger.warning(
-                    "`JobQueueManager` can not be initialized. This might be due to missing dependencies (APScheduler), invalid configuration or backend not being available."
-                )
-                return None
-
-        else:
-            raise ImportError(
-                f"Invalid job queue type: {type}. Valid types: ['rq', 'apscheduler']"
-            )
-
-
-__all__ = [
-    "JobQueueManager",
-    "RQManager",
-    "APSManager",
-    # "HueyWorker",
-    "JobQueueBackend",
-    "RQBackend",
-    "APSBackend",
-]

flowerpower/job_queue/apscheduler/__init__.py

@@ -1,11 +0,0 @@
-from .manager import APSManager
-from .setup import APSBackend, APSDataStore, APSEventBroker
-from .trigger import APSTrigger
-
-__all__ = [
-    "APSManager",
-    "APSTrigger",
-    "APSBackend",
-    "APSDataStore",
-    "APSEventBroker",
-]

flowerpower/job_queue/apscheduler/_setup/datastore.py

@@ -1,110 +0,0 @@
-from apscheduler.datastores.base import BaseDataStore
-from sqlalchemy import text
-from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
-
-from ...base import BackendType, BaseBackend
-
-
-class APSDataStoreType(BackendType):
-    POSTGRESQL = "postgresql"
-    SQLITE = "sqlite"
-    MYSQL = "mysql"
-    MONGODB = "mongodb"
-    MEMORY = "memory"
-
-
-class APSDataStore(BaseBackend):
-    """Data store for APScheduler."""
-
-    def __post_init__(self):
-        super().__post_init__(backend_type=APSDataStoreType)
-        self._validate_inputs()
-
-    @classmethod
-    def from_dict(cls, d: dict[str, any]) -> "APSDataStore":
-        return cls(**d)
-
-    def _validate_inputs(self) -> None:
-        if self.type.value not in [ds.value for ds in APSDataStoreType]:
-            raise ValueError(
-                f"Invalid data store type: {self.type}. Valid types: {[ds.value for ds in APSDataStoreType]}"
-            )
-
-    async def _setup_db(self) -> None:
-        sqla_engine = create_async_engine(self.uri)
-
-        try:
-            await self._create_schema(sqla_engine)
-        except Exception:
-            await self._create_database_and_schema(sqla_engine)
-
-    async def _create_schema(self, engine: AsyncEngine) -> None:
-        if not self.schema_or_queue:
-            return
-
-        async with engine.begin() as conn:
-            await conn.execute(
-                text(f"CREATE SCHEMA IF NOT EXISTS {self.schema_or_queue}")
-            )
-            await conn.commit()
-
-    async def _create_database_and_schema(self, engine: AsyncEngine) -> None:
-        database_name = self.uri.split("/")[-1].split("?")[0]
-        temp_uri = self.uri.replace(f"/{database_name}", "/template1")
-        temp_engine = create_async_engine(temp_uri)
-
-        async with temp_engine.begin() as conn:
-            await conn.execute(text("COMMIT"))
-            try:
-                await conn.execute(text(f"CREATE DATABASE {database_name}"))
-            finally:
-                await conn.execute(text("COMMIT"))
-
-        if self.schema_or_queue:
-            await self._create_schema(engine)
-
-    def setup_db(self) -> None:
-        from anyio.from_thread import start_blocking_portal
-
-        with start_blocking_portal() as portal:
-            portal.call(self._setup_db)
-
-    def _setup_sqlalchemy(self) -> None:
-        from apscheduler.datastores.sqlalchemy import SQLAlchemyDataStore
-
-        if not self.type.is_sqlite_type:
-            self.setup_db()
-        self._sqla_engine = create_async_engine(self.uri)
-        self._client = SQLAlchemyDataStore(
-            self._sqla_engine, schema=self.schema_or_queue
-        )
-
-    def _setup_mongodb(self) -> None:
-        from apscheduler.datastores.mongodb import MongoDBDataStore
-
-        self._client = MongoDBDataStore(self.uri, database=self.schema_or_queue)
-
-    def _setup_memory(self) -> None:
-        from apscheduler.datastores.memory import MemoryDataStore
-
-        self._client = MemoryDataStore()
-
-    def setup(self) -> None:
-        if self.type.is_sqla_type:
-            self._setup_sqlalchemy()
-        elif self.type.is_mongodb_type:
-            self._setup_mongodb()
-        else:
-            self._setup_memory()
-
-    @property
-    def client(self) -> BaseDataStore:
-        if self._client is None:
-            self.setup()
-        return self._client
-
-    @property
-    def sqla_engine(self) -> AsyncEngine | None:
-        if self._sqla_engine is None:
-            self.setup()
-        return self._sqla_engine

flowerpower/job_queue/apscheduler/_setup/eventbroker.py

@@ -1,93 +0,0 @@
-from apscheduler.eventbrokers.base import BaseEventBroker
-from sqlalchemy.ext.asyncio import AsyncEngine
-
-from ...base import BackendType, BaseBackend
-
-
-class APSEventBrokerType(BackendType):
-    POSTGRESQL = "postgresql"
-    MEMORY = "memory"
-    REDIS = "redis"
-    MQTT = "mqtt"
-
-
-class APSEventBroker(BaseBackend):
-    """Data store for APScheduler."""
-
-    def __post_init__(self):
-        super().__post_init__(backend_type=APSEventBrokerType)
-
-    @classmethod
-    def from_dict(cls, d: dict[str, any]) -> "APSEventBroker":
-        return cls(**d)
-
-    def _validate_inputs(self) -> None:
-        if self.type.value not in [ds.value for ds in APSEventBrokerType]:
-            raise ValueError(
-                f"Invalid data store type: {self.type}. Valid types: {[ds.value for ds in APSEventBrokerType]}"
-            )
-
-    def _setup_asyncpg_event_broker(self):
-        from apscheduler.eventbrokers.asyncpg import AsyncpgEventBroker
-
-        if self._sqla_engine is None:
-            self._event_broker = AsyncpgEventBroker.from_dsn(dsn=self.uri)
-        else:
-            self._event_broker = AsyncpgEventBroker.from_async_sqla_engine(
-                engine=self._sqla_engine
-            )
-
-    def _setup_mqtt_event_broker(self):
-        import urllib.parse
-
-        from apscheduler.eventbrokers.mqtt import MQTTEventBroker
-
-        # Parse the URI
-        parsed = urllib.parse.urlparse(self.uri)
-
-        hostname = parsed.hostname
-        port = parsed.port
-        username = parsed.username
-        password = parsed.password
-        use_ssl = parsed.scheme == "mqtts"
-
-        self._event_broker = MQTTEventBroker(
-            host=hostname, port=port, ssl=use_ssl, topic="flowerpower/scheduler"
-        )
-        if (self.username is not None) and (self.password is not None):
-            self._event_broker._client.username_pw_set(
-                username,
-                password,
-            )
-
-    def _setup_redis_event_broker(self):
-        from apscheduler.eventbrokers.redis import RedisEventBroker
-
-        self._event_broker = RedisEventBroker(self.uri)
-
-    def _setup_local_event_broker(self):
-        from apscheduler.eventbrokers.local import LocalEventBroker
-
-        self._event_broker = LocalEventBroker()
-
-    def setup(self):
-        if self.is_sqla_type:
-            self._setup_asyncpg_event_broker()
-        elif self.is_mqtt_type:
-            self._setup_mqtt_event_broker()
-        elif self.is_redis_type:
-            self._setup_redis_event_broker()
-        else:
-            self._setup_local_event_broker()
-
-    @property
-    def client(self) -> BaseEventBroker:
-        if self._event_broker is None:
-            self.setup()
-        return self._event_broker
-
-    @property
-    def sqla_engine(self) -> AsyncEngine | None:
-        if self._sqla_engine is None:
-            self.setup()
-        return self._sqla_engine