FlowerPower 0.9.13.1__py3-none-any.whl → 1.0.0b2__py3-none-any.whl

flowerpower/io/base.py

@@ -1,6 +1,6 @@
 import importlib
 import posixpath
-from typing import Generator, Any
+from typing import Any, Generator
 
 import datafusion
 import duckdb
@@ -12,23 +12,14 @@ from fsspec.utils import get_protocol
 from pydantic import BaseModel, ConfigDict
 
 from ..fs import get_filesystem
-from ..fs.ext import path_to_glob, _dict_to_dataframe
-from ..fs.storage_options import (
-    AwsStorageOptions,
-    AzureStorageOptions,
-    GcsStorageOptions,
-    GitHubStorageOptions,
-    GitLabStorageOptions,
-    StorageOptions,
-)
+from ..fs.ext import _dict_to_dataframe, path_to_glob
+from ..fs.storage_options import (AwsStorageOptions, AzureStorageOptions,
+                                  GcsStorageOptions, GitHubStorageOptions,
+                                  GitLabStorageOptions, StorageOptions)
 from ..utils.misc import convert_large_types_to_standard, to_pyarrow_table
 from ..utils.polars import pl
 from ..utils.sql import sql2polars_filter, sql2pyarrow_filter
-from .metadata import (
-    get_dataframe_metadata,
-    get_pyarrow_dataset_metadata,
-)
-
+from .metadata import get_dataframe_metadata, get_pyarrow_dataset_metadata
 
 if importlib.util.find_spec("pydala"):
     from pydala.dataset import ParquetDataset
@@ -1420,9 +1411,13 @@ class BaseDatabaseIO(BaseModel):
             db in ["postgres", "mysql", "mssql", "oracle"]
             and not self.connection_string
         ):
-            if not all(
-                [self.username, self.password, self.server, self.port, self.database]
-            ):
+            if not all([
+                self.username,
+                self.password,
+                self.server,
+                self.port,
+                self.database,
+            ]):
                 raise ValueError(
                     f"{self.type_} requires connection_string or username, password, server, port, and table_name "
                     "to build it."

flowerpower/io/loader/__init__.py

@@ -0,0 +1,28 @@
+from .csv import CSVDatasetReader, CSVFileReader
+from .deltatable import DeltaTableReader
+from .duckdb import DuckDBReader
+from .json import JsonDatasetReader, JsonFileReader
+from .mssql import MSSQLReader
+from .mysql import MySQLReader
+from .oracle import OracleDBReader
+from .parquet import ParquetDatasetReader, ParquetFileReader
+from .postgres import PostgreSQLReader
+from .pydala import PydalaDatasetReader
+from .sqlite import SQLiteReader
+
+__all__ = [
+    "CSVFileReader",
+    "CSVDatasetReader",
+    "DeltaTableReader",
+    "DuckDBReader",
+    "JsonFileReader",
+    "JsonDatasetReader",
+    "MSSQLReader",
+    "MySQLReader",
+    "OracleDBReader",
+    "ParquetFileReader",
+    "ParquetDatasetReader",
+    "PostgreSQLReader",
+    "PydalaDatasetReader",
+    "SQLiteReader",
+]

flowerpower/io/loader/deltatable.py

@@ -1,26 +1,23 @@
 # import datetime as dt
 
 
+import datetime
+
 import pyarrow as pa
 import pyarrow.dataset as pds
 from deltalake import DeltaTable, table
 from deltalake.exceptions import TableNotFoundError
-import datetime
+# from ..utils import get_dataframe_metadata, get_delta_metadata
+from loguru import logger
 from sherlock import RedisLock
+
 from ..base import BaseDatasetReader
-from ..metadata import (
-    get_delta_metadata,
-    get_dataframe_metadata,
-    get_pyarrow_dataset_metadata,
-)
+from ..metadata import (get_dataframe_metadata, get_delta_metadata,
+                        get_pyarrow_dataset_metadata)
 
 # from hamilton.function_modifiers import dataloader
 
 
-# from ..utils import get_dataframe_metadata, get_delta_metadata
-from loguru import logger
-
-
 class DeltaTableReader(BaseDatasetReader):
     """Delta table loader.
 

flowerpower/io/metadata.py

@@ -8,6 +8,7 @@ import pyarrow as pa
 import pyarrow.dataset as pds
 from deltalake import DeltaTable
 from fsspec import AbstractFileSystem
+
 from ..fs.ext import path_to_glob
 
 

flowerpower/io/saver/__init__.py

@@ -0,0 +1,28 @@
+from .csv import CSVDatasetWriter, CSVFileWriter
+from .deltatable import DeltaTableWriter
+from .duckdb import DuckDBWriter
+from .json import JsonDatasetWriter, JsonFileWriter
+from .mssql import MSSQLWriter
+from .mysql import MySQLWriter
+from .oracle import OracleDBWriter
+from .parquet import ParquetDatasetWriter, ParquetFileWriter
+from .postgres import PostgreSQLWriter
+from .pydala import PydalaDatasetWriter
+from .sqlite import SQLiteWriter
+
+__all__ = [
+    "CSVFileWriter",
+    "CSVDatasetWriter",
+    "DeltaTableWriter",
+    "DuckDBWriter",
+    "JsonFileWriter",
+    "JsonDatasetWriter",
+    "MSSQLWriter",
+    "MySQLWriter",
+    "OracleDBWriter",
+    "ParquetFileWriter",
+    "ParquetDatasetWriter",
+    "PostgreSQLWriter",
+    "PydalaDatasetWriter",
+    "SQLiteWriter",
+]

flowerpower/io/saver/deltatable.py

@@ -3,14 +3,15 @@ from typing import Any
 import pandas as pd
 import polars as pl
 import pyarrow as pa
-from deltalake.table import ColumnProperties, CommitProperties, PostCommitHookProperties
+from deltalake.table import (ColumnProperties, CommitProperties,
+                             PostCommitHookProperties)
 from deltalake.writer import WriterProperties, write_deltalake
+from redis import Redis, StrictRedis
+from sherlock import RedisLock
 
 from ...utils.misc import _dict_to_dataframe
 from ..base import BaseDatasetWriter
 from ..metadata import get_dataframe_metadata
-from sherlock import RedisLock
-from redis import StrictRedis, Redis
 
 
 class DeltaTableWriter(BaseDatasetWriter):

flowerpower/job_queue/__init__.py

@@ -0,0 +1,252 @@
+from typing import Any, Optional
+
+from ..fs import AbstractFileSystem
+from ..utils.logging import setup_logging
+from .apscheduler import APSBackend, APSManager
+from .base import BaseBackend, BaseJobQueueManager
+from .rq import RQBackend, RQManager
+from ..cfg.project import ProjectConfig
+
+setup_logging()
+
+
+class JobQueue:
+    """A factory class for creating job queue instances for job scheduling and execution.
+
+    This class provides a unified interface for creating different types of job queue instances
+    (RQ, APScheduler, Huey) based on the specified backend type. Each job queue type provides
+    different capabilities for job scheduling and execution.
+
+    The job queue instances handle:
+    - Job scheduling and execution
+    - Background task processing
+    - Job queue management
+    - Result storage and retrieval
+
+    Example:
+        ```python
+        # Create an RQ job queue
+        rq_worker = JobQueue(
+            type="rq",
+            name="my_worker",
+            log_level="DEBUG"
+        )
+
+        # Create an APScheduler job queue with custom backend
+        from flowerpower.job_queue.apscheduler import APSBackend
+        backend_config = APSBackend(
+            data_store={"type": "postgresql", "uri": "postgresql+asyncpg://user:pass@localhost/db"},
+            event_broker={"type": "redis", "uri": "redis://localhost:6379/0"}
+        )
+        aps_worker = JobQueue(
+            type="apscheduler",
+            name="scheduler",
+            backend=backend_config
+        )
+
+        ```
+    """
+
+    def __new__(
+        cls,
+        type: str |None = None,
+        name: str | None = None,
+        base_dir: str | None = ".",
+        backend: BaseBackend | None = None,
+        storage_options: Optional[dict[str, Any]] = None,
+        fs: AbstractFileSystem | None = None,
+        log_level: str | None = None,
+        **kwargs,
+    ) -> BaseJobQueueManager:
+        """Create a new job queue instance based on the specified backend type.
+
+        Args:
+            type: The type of job queue to create. Valid values are:
+                - "rq": Redis Queue job queue for Redis-based job queuing
+                - "apscheduler": APScheduler job queue for advanced job scheduling
+            name: Name of the job queue instance. Used for identification in logs
+                and monitoring.
+            base_dir: Base directory for job queue files and configuration. Defaults
+                to current working directory if not specified.
+            backend: Pre-configured backend instance. If not provided, one will
+                be created based on configuration settings.
+            storage_options: Options for configuring filesystem storage access.
+                Example: {"mode": "async", "root": "/tmp", "protocol": "s3"}
+            fs: Custom filesystem implementation for storage operations.
+                Example: S3FileSystem, LocalFileSystem, etc.
+            log_level: Logging level for the job queue. Valid values are:
+                "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
+            **kwargs: Additional configuration options passed to the specific
+                job queue implementation.
+
+        Returns:
+            BaseJobQueueManager: An instance of the specified job queue type (RQManager,
+                APSManager).
+
+        Raises:
+            ValueError: If an invalid job queue type is specified.
+            ImportError: If required dependencies for the chosen job queue type
+                are not installed.
+            RuntimeError: If job queue initialization fails due to configuration
+                or connection issues.
+
+        Example:
+            ```python
+            # Basic RQ job queue
+            worker = JobQueue(type="rq", name="basic_worker")
+
+            # APScheduler with custom logging and storage
+            worker = JobQueue(
+                type="apscheduler",
+                name="scheduler",
+                base_dir="/app/data",
+                storage_options={"mode": "async"},
+                log_level="DEBUG"
+            )
+
+            ```
+        """
+        if type is None:
+            type = ProjectConfig.load(
+                base_dir=base_dir, name=name, fs=fs, storage_options=storage_options or {}).job_queue.type
+            
+        if type == "rq":
+            return RQManager(
+                name=name,
+                base_dir=base_dir,
+                backend=backend,
+                storage_options=storage_options,
+                fs=fs,
+                log_level=log_level,
+                **kwargs,
+            )
+        elif type == "apscheduler":
+            return APSManager(
+                name=name,
+                base_dir=base_dir,
+                backend=backend,
+                storage_options=storage_options,
+                fs=fs,
+                log_level=log_level,
+                **kwargs,
+            )
+        
+        else:
+            raise ValueError(
+                f"Invalid job queue type: {type}. Valid types: ['rq', 'apscheduler']"
+            )
+
+
+class Backend:
+    """A factory class for creating backend instances for different job queue types.
+
+    This class provides a unified interface for creating backend instances that handle
+    the storage, queuing, and event management for different job queue types. Each backend
+    type provides specific implementations for:
+    - Job storage and persistence
+    - Queue management
+    - Event handling and communication
+    - Result storage
+
+    Example:
+        ```python
+        # Create RQ backend with Redis
+        rq_backend = Backend(
+            job_queue_type="rq",
+            uri="redis://localhost:6379/0",
+            queues=["high", "default", "low"]
+        )
+
+        # Create APScheduler backend with PostgreSQL and Redis
+        aps_backend = Backend(
+            job_queue_type="apscheduler",
+            data_store={
+                "type": "postgresql",
+                "uri": "postgresql+asyncpg://user:pass@localhost/db"
+            },
+            event_broker={
+                "type": "redis",
+                "uri": "redis://localhost:6379/0"
+            }
+        )
+        ```
+    """
+
+    def __new__(
+        cls,
+        job_queue_type: str,
+        **kwargs,
+    ) -> BaseBackend:
+        """Create a new backend instance based on the specified job queue type.
+
+        Args:
+            job_queue_type: The type of backend to create. Valid values are:
+                - "rq": Redis Queue backend using Redis
+                - "apscheduler": APScheduler backend supporting various databases
+                    and event brokers
+            **kwargs: Backend-specific configuration options:
+                For RQ:
+                    - uri (str): Redis connection URI
+                    - queues (list[str]): List of queue names
+                    - result_ttl (int): Time to live for results in seconds
+                For APScheduler:
+                    - data_store (dict): Data store configuration
+                    - event_broker (dict): Event broker configuration
+                    - cleanup_interval (int): Cleanup interval in seconds
+                    - max_concurrent_jobs (int): Maximum concurrent jobs
+
+        Returns:
+            BaseBackend: An instance of RQBackend or APSBackend depending on
+                the specified job queue type.
+
+        Raises:
+            ValueError: If an invalid job queue type is specified.
+            RuntimeError: If backend initialization fails due to configuration
+                or connection issues.
+
+        Example:
+            ```python
+            # Create RQ backend
+            rq_backend = Backend(
+                job_queue_type="rq",
+                uri="redis://localhost:6379/0",
+                queues=["high", "default", "low"],
+                result_ttl=3600
+            )
+
+            # Create APScheduler backend with PostgreSQL and Redis
+            aps_backend = Backend(
+                job_queue_type="apscheduler",
+                data_store={
+                    "type": "postgresql",
+                    "uri": "postgresql+asyncpg://user:pass@localhost/db",
+                    "schema": "scheduler"
+                },
+                event_broker={
+                    "type": "redis",
+                    "uri": "redis://localhost:6379/0"
+                },
+                cleanup_interval=300,
+                max_concurrent_jobs=10
+            )
+            ```
+        """
+        if job_queue_type == "rq":
+            return RQBackend(**kwargs)
+        elif job_queue_type == "apscheduler":
+            return APSBackend(**kwargs)
+        else:
+            raise ValueError(
+                f"Invalid job queue type: {job_queue_type}. Valid types: ['rq', 'apscheduler']"
+            )
+
+
+__all__ = [
+    "JobQueue",
+    "RQManager",
+    "APSManager",
+    #"HueyWorker",
+    "Backend",
+    "RQBackend",
+    "APSBackend",
+]

flowerpower/job_queue/apscheduler/__init__.py

@@ -0,0 +1,11 @@
+from .setup import APSBackend, APSDataStore, APSEventBroker
+from .trigger import APSTrigger
+from .manager import APSManager
+
+__all__ = [
+    "APSManager",
+    "APSTrigger",
+    "APSBackend",
+    "APSDataStore",
+    "APSEventBroker",
+]

flowerpower/job_queue/apscheduler/_setup/datastore.py

@@ -0,0 +1,110 @@
+from apscheduler.datastores.base import BaseDataStore
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
+
+from ...base import BackendType, BaseBackend
+
+
+class APSDataStoreType(BackendType):
+    POSTGRESQL = "postgresql"
+    SQLITE = "sqlite"
+    MYSQL = "mysql"
+    MONGODB = "mongodb"
+    MEMORY = "memory"
+
+
+class APSDataStore(BaseBackend):
+    """Data store for APScheduler."""
+
+    def __post_init__(self):
+        super().__post_init__(backend_type=APSDataStoreType)
+        self._validate_inputs()
+
+    @classmethod
+    def from_dict(cls, d: dict[str, any]) -> "APSDataStore":
+        return cls(**d)
+
+    def _validate_inputs(self) -> None:
+        if self.type.value not in [ds.value for ds in APSDataStoreType]:
+            raise ValueError(
+                f"Invalid data store type: {self.type}. Valid types: {[ds.value for ds in APSDataStoreType]}"
+            )
+
+    async def _setup_db(self) -> None:
+        sqla_engine = create_async_engine(self.uri)
+
+        try:
+            await self._create_schema(sqla_engine)
+        except Exception:
+            await self._create_database_and_schema(sqla_engine)
+
+    async def _create_schema(self, engine: AsyncEngine) -> None:
+        if not self.schema_or_queue:
+            return
+
+        async with engine.begin() as conn:
+            await conn.execute(
+                text(f"CREATE SCHEMA IF NOT EXISTS {self.schema_or_queue}")
+            )
+            await conn.commit()
+
+    async def _create_database_and_schema(self, engine: AsyncEngine) -> None:
+        database_name = self.uri.split("/")[-1].split("?")[0]
+        temp_uri = self.uri.replace(f"/{database_name}", "/template1")
+        temp_engine = create_async_engine(temp_uri)
+
+        async with temp_engine.begin() as conn:
+            await conn.execute(text("COMMIT"))
+            try:
+                await conn.execute(text(f"CREATE DATABASE {database_name}"))
+            finally:
+                await conn.execute(text("COMMIT"))
+
+        if self.schema_or_queue:
+            await self._create_schema(engine)
+
+    def setup_db(self) -> None:
+        from anyio.from_thread import start_blocking_portal
+
+        with start_blocking_portal() as portal:
+            portal.call(self._setup_db)
+
+    def _setup_sqlalchemy(self) -> None:
+        from apscheduler.datastores.sqlalchemy import SQLAlchemyDataStore
+
+        if not self.type.is_sqlite_type:
+            self.setup_db()
+        self._sqla_engine = create_async_engine(self.uri)
+        self._client = SQLAlchemyDataStore(
+            self._sqla_engine, schema=self.schema_or_queue
+        )
+
+    def _setup_mongodb(self) -> None:
+        from apscheduler.datastores.mongodb import MongoDBDataStore
+
+        self._client = MongoDBDataStore(self.uri, database=self.schema_or_queue)
+
+    def _setup_memory(self) -> None:
+        from apscheduler.datastores.memory import MemoryDataStore
+
+        self._client = MemoryDataStore()
+
+    def setup(self) -> None:
+        if self.type.is_sqla_type:
+            self._setup_sqlalchemy()
+        elif self.type.is_mongodb_type:
+            self._setup_mongodb()
+        else:
+            self._setup_memory()
+
+    @property
+    def client(self) -> BaseDataStore:
+        if self._client is None:
+            self.setup()
+        return self._client
+
+    @property
+    def sqla_engine(self) -> AsyncEngine | None:
+        if self._sqla_engine is None:
+            self.setup()
+        return self._sqla_engine

flowerpower/job_queue/apscheduler/_setup/eventbroker.py

@@ -0,0 +1,93 @@
+from apscheduler.eventbrokers.base import BaseEventBroker
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+from ...base import BackendType, BaseBackend
+
+
+class APSEventBrokerType(BackendType):
+    POSTGRESQL = "postgresql"
+    MEMORY = "memory"
+    REDIS = "redis"
+    MQTT = "mqtt"
+
+
+class APSEventBroker(BaseBackend):
+    """Data store for APScheduler."""
+
+    def __post_init__(self):
+        super().__post_init__(backend_type=APSEventBrokerType)
+
+    @classmethod
+    def from_dict(cls, d: dict[str, any]) -> "APSEventBroker":
+        return cls(**d)
+
+    def _validate_inputs(self) -> None:
+        if self.type.value not in [ds.value for ds in APSEventBrokerType]:
+            raise ValueError(
+                f"Invalid data store type: {self.type}. Valid types: {[ds.value for ds in APSEventBrokerType]}"
+            )
+
+    def _setup_asyncpg_event_broker(self):
+        from apscheduler.eventbrokers.asyncpg import AsyncpgEventBroker
+
+        if self._sqla_engine is None:
+            self._event_broker = AsyncpgEventBroker.from_dsn(dsn=self.uri)
+        else:
+            self._event_broker = AsyncpgEventBroker.from_async_sqla_engine(
+                engine=self._sqla_engine
+            )
+
+    def _setup_mqtt_event_broker(self):
+        import urllib.parse
+
+        from apscheduler.eventbrokers.mqtt import MQTTEventBroker
+
+        # Parse the URI
+        parsed = urllib.parse.urlparse(self.uri)
+
+        hostname = parsed.hostname
+        port = parsed.port
+        username = parsed.username
+        password = parsed.password
+        use_ssl = parsed.scheme == "mqtts"
+
+        self._event_broker = MQTTEventBroker(
+            host=hostname, port=port, ssl=use_ssl, topic="flowerpower/scheduler"
+        )
+        if (self.username is not None) and (self.password is not None):
+            self._event_broker._client.username_pw_set(
+                username,
+                password,
+            )
+
+    def _setup_redis_event_broker(self):
+        from apscheduler.eventbrokers.redis import RedisEventBroker
+
+        self._event_broker = RedisEventBroker(self.uri)
+
+    def _setup_local_event_broker(self):
+        from apscheduler.eventbrokers.local import LocalEventBroker
+
+        self._event_broker = LocalEventBroker()
+
+    def setup(self):
+        if self.is_sqla_type:
+            self._setup_asyncpg_event_broker()
+        elif self.is_mqtt_type:
+            self._setup_mqtt_event_broker()
+        elif self.is_redis_type:
+            self._setup_redis_event_broker()
+        else:
+            self._setup_local_event_broker()
+
+    @property
+    def client(self) -> BaseEventBroker:
+        if self._event_broker is None:
+            self.setup()
+        return self._event_broker
+
+    @property
+    def sqla_engine(self) -> AsyncEngine | None:
+        if self._sqla_engine is None:
+            self.setup()
+        return self._sqla_engine