FlowerPower 1.0.0b1__py3-none-any.whl → 1.0.0b3__py3-none-any.whl

flowerpower/flowerpower.py

@@ -4,16 +4,16 @@ import posixpath
 from pathlib import Path
 
 import rich
-from fsspec.spec import AbstractFileSystem
 
-from .cfg import ProjectConfig
-from .fs import get_filesystem
 from . import settings
+from .cfg import ProjectConfig
+from .fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
+
 
 def init(
     name: str | None = None,
     base_dir: str | None = None,
-    storage_options: dict = {},
+    storage_options: dict | BaseStorageOptions | None = {},
     fs: AbstractFileSystem | None = None,
     job_queue_type: str = settings.DEFAULT_JOB_QUEUE,
     cfg_dir: str = settings.CONFIG_DIR,
@@ -27,20 +27,26 @@ def init(
     if base_dir is None:
         base_dir = str(Path.cwd())
 
-    fs = get_filesystem(posixpath.join(base_dir, name), **storage_options)
+    if fs is None:
+        fs = get_filesystem(
+            posixpath.join(base_dir, name),
+            cached=True,
+            dirfs=True,
+            storage_options=storage_options,
+        )
 
     fs.makedirs(f"{cfg_dir}/pipelines", exist_ok=True)
     fs.makedirs(pipelines_dir, exist_ok=True)
     fs.makedirs(hooks_dir, exist_ok=True)
 
-    cfg = ProjectConfig.load(base_dir=posixpath.join(base_dir, name), name=name, job_queue_type=job_queue_type)
+    cfg = ProjectConfig.load(name=name, job_queue_type=job_queue_type, fs=fs)
 
-    with open(posixpath.join(base_dir, name, "README.md"), "w") as f:
+    with fs.open("README.md", "w") as f:
         f.write(
             f"# {name.replace('_', ' ').upper()}\n\n"
             f"**created with FlowerPower**\n\n*{dt.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*\n\n"
         )
-    cfg.save()
+    cfg.save(fs=fs)
     os.chdir(posixpath.join(base_dir, name))
 
     rich.print(

flowerpower/fs/__init__.py

@@ -11,9 +11,13 @@ else:
 from .base import get_filesystem  # noqa: E402
 from .storage_options import AwsStorageOptions  # noqa: E402
 from .storage_options import AzureStorageOptions  # noqa: E402
-from .storage_options import (BaseStorageOptions, GcsStorageOptions,
-                              GitHubStorageOptions, GitLabStorageOptions,
-                              StorageOptions)
+from .storage_options import (
+    BaseStorageOptions,
+    GcsStorageOptions,
+    GitHubStorageOptions,
+    GitLabStorageOptions,
+    StorageOptions,
+)
 
 __all__ = [
     "get_filesystem",

flowerpower/fs/ext.py

@@ -11,8 +11,12 @@ import pyarrow.dataset as pds
 import pyarrow.parquet as pq
 from fsspec import AbstractFileSystem
 
-from ..utils.misc import (_dict_to_dataframe, convert_large_types_to_standard,
-                          run_parallel, to_pyarrow_table)
+from ..utils.misc import (
+    _dict_to_dataframe,
+    convert_large_types_to_standard,
+    run_parallel,
+    to_pyarrow_table,
+)
 from ..utils.polars import pl
 
 if importlib.util.find_spec("duckdb") is not None:

flowerpower/io/base.py

@@ -13,9 +13,14 @@ from pydantic import BaseModel, ConfigDict
 
 from ..fs import get_filesystem
 from ..fs.ext import _dict_to_dataframe, path_to_glob
-from ..fs.storage_options import (AwsStorageOptions, AzureStorageOptions,
-                                  GcsStorageOptions, GitHubStorageOptions,
-                                  GitLabStorageOptions, StorageOptions)
+from ..fs.storage_options import (
+    AwsStorageOptions,
+    AzureStorageOptions,
+    GcsStorageOptions,
+    GitHubStorageOptions,
+    GitLabStorageOptions,
+    StorageOptions,
+)
 from ..utils.misc import convert_large_types_to_standard, to_pyarrow_table
 from ..utils.polars import pl
 from ..utils.sql import sql2polars_filter, sql2pyarrow_filter
@@ -1411,13 +1416,15 @@ class BaseDatabaseIO(BaseModel):
             db in ["postgres", "mysql", "mssql", "oracle"]
             and not self.connection_string
         ):
-            if not all([
-                self.username,
-                self.password,
-                self.server,
-                self.port,
-                self.database,
-            ]):
+            if not all(
+                [
+                    self.username,
+                    self.password,
+                    self.server,
+                    self.port,
+                    self.database,
+                ]
+            ):
                 raise ValueError(
                     f"{self.type_} requires connection_string or username, password, server, port, and table_name "
                     "to build it."

flowerpower/io/loader/_duckdb.py

@@ -6,6 +6,7 @@ import pandas as pd
 import polars as pl
 import pyarrow as pa
 import pyarrow.dataset as pds
+
 # from hamilton.function_modifiers import dataloader
 from pydantic import BaseModel
 

flowerpower/io/loader/deltatable.py

@@ -7,13 +7,17 @@ import pyarrow as pa
 import pyarrow.dataset as pds
 from deltalake import DeltaTable, table
 from deltalake.exceptions import TableNotFoundError
+
 # from ..utils import get_dataframe_metadata, get_delta_metadata
 from loguru import logger
 from sherlock import RedisLock
 
 from ..base import BaseDatasetReader
-from ..metadata import (get_dataframe_metadata, get_delta_metadata,
-                        get_pyarrow_dataset_metadata)
+from ..metadata import (
+    get_dataframe_metadata,
+    get_delta_metadata,
+    get_pyarrow_dataset_metadata,
+)
 
 # from hamilton.function_modifiers import dataloader
 

flowerpower/io/saver/deltatable.py

@@ -3,8 +3,7 @@ from typing import Any
 import pandas as pd
 import polars as pl
 import pyarrow as pa
-from deltalake.table import (ColumnProperties, CommitProperties,
-                             PostCommitHookProperties)
+from deltalake.table import ColumnProperties, CommitProperties, PostCommitHookProperties
 from deltalake.writer import WriterProperties, write_deltalake
 from redis import Redis, StrictRedis
 from sherlock import RedisLock

flowerpower/job_queue/__init__.py

@@ -1,16 +1,16 @@
 from typing import Any, Optional
 
+from ..cfg.project import ProjectConfig
 from ..fs import AbstractFileSystem
 from ..utils.logging import setup_logging
 from .apscheduler import APSBackend, APSManager
 from .base import BaseBackend, BaseJobQueueManager
 from .rq import RQBackend, RQManager
-from ..cfg.project import ProjectConfig
 
 setup_logging()
 
 
-class JobQueue:
+class JobQueueManager:
     """A factory class for creating job queue instances for job scheduling and execution.
 
     This class provides a unified interface for creating different types of job queue instances
@@ -26,7 +26,7 @@ class JobQueue:
     Example:
         ```python
         # Create an RQ job queue
-        rq_worker = JobQueue(
+        rq_worker = JobQueueManager(
             type="rq",
             name="my_worker",
             log_level="DEBUG"
@@ -38,7 +38,7 @@ class JobQueue:
             data_store={"type": "postgresql", "uri": "postgresql+asyncpg://user:pass@localhost/db"},
             event_broker={"type": "redis", "uri": "redis://localhost:6379/0"}
         )
-        aps_worker = JobQueue(
+        aps_worker = JobQueueManager(
             type="apscheduler",
             name="scheduler",
             backend=backend_config
@@ -49,7 +49,7 @@ class JobQueue:
 
     def __new__(
         cls,
-        type: str |None = None,
+        type: str | None = None,
         name: str | None = None,
         base_dir: str | None = ".",
         backend: BaseBackend | None = None,
@@ -93,10 +93,10 @@ class JobQueue:
         Example:
             ```python
             # Basic RQ job queue
-            worker = JobQueue(type="rq", name="basic_worker")
+            worker = JobQueueManager(type="rq", name="basic_worker")
 
             # APScheduler with custom logging and storage
-            worker = JobQueue(
+            worker = JobQueueManager(
                 type="apscheduler",
                 name="scheduler",
                 base_dir="/app/data",
@@ -108,8 +108,12 @@ class JobQueue:
         """
         if type is None:
             type = ProjectConfig.load(
-                base_dir=base_dir, name=name, fs=fs, storage_options=storage_options or {}).job_queue.type
-            
+                base_dir=base_dir,
+                name=name,
+                fs=fs,
+                storage_options=storage_options or {},
+            ).job_queue.type
+
         if type == "rq":
             return RQManager(
                 name=name,
@@ -130,7 +134,7 @@ class JobQueue:
                 log_level=log_level,
                 **kwargs,
             )
-        
+
         else:
             raise ValueError(
                 f"Invalid job queue type: {type}. Valid types: ['rq', 'apscheduler']"
@@ -242,10 +246,10 @@ class Backend:
 
 
 __all__ = [
-    "JobQueue",
+    "JobQueueManager",
     "RQManager",
     "APSManager",
-    #"HueyWorker",
+    # "HueyWorker",
     "Backend",
     "RQBackend",
     "APSBackend",

flowerpower/job_queue/apscheduler/__init__.py

@@ -1,6 +1,6 @@
+from .manager import APSManager
 from .setup import APSBackend, APSDataStore, APSEventBroker
 from .trigger import APSTrigger
-from .manager import APSManager
 
 __all__ = [
     "APSManager",

flowerpower/job_queue/apscheduler/manager.py

@@ -8,6 +8,7 @@ import datetime as dt
 import importlib.util
 from typing import Any, Callable
 from uuid import UUID
+
 import duration_parser
 from fsspec.spec import AbstractFileSystem
 from loguru import logger
@@ -181,11 +182,13 @@ class APSManager(BaseJobQueueManager):
                     sqla_engine=data_store.sqla_engine
                 )
             else:
-                event_broker = APSEventBroker(**{
-                    k: v
-                    for k, v in self.cfg.backend.event_broker.to_dict().items()
-                    if k != "from_ds_sqla"
-                })
+                event_broker = APSEventBroker(
+                    **{
+                        k: v
+                        for k, v in self.cfg.backend.event_broker.to_dict().items()
+                        if k != "from_ds_sqla"
+                    }
+                )
             self._backend = APSBackend(data_store=data_store, event_broker=event_broker)
 
         logger.info(
@@ -375,7 +378,9 @@ class APSManager(BaseJobQueueManager):
         if isinstance(result_ttl, (int, float)):
             result_ttl = dt.timedelta(seconds=result_ttl)
 
-        run_at = dt.datetime.fromisoformat(run_at) if isinstance(run_at, str) else run_at
+        run_at = (
+            dt.datetime.fromisoformat(run_at) if isinstance(run_at, str) else run_at
+        )
         run_in = duration_parser.parse(run_in) if isinstance(run_in, str) else run_in
 
         if run_in:

flowerpower/job_queue/apscheduler/utils.py

@@ -160,10 +160,12 @@ def format_trigger(trigger):
             )
             cron_parts = {k: v.strip("'") for k, v in cron_parts.items()}
             crontab = f"{cron_parts['minute']} {cron_parts['hour']} {cron_parts['day']} {cron_parts['month']} {cron_parts['day_of_week']}"
-            human_readable = humanize_crontab(**{
-                k: cron_parts[k]
-                for k in ["minute", "hour", "day", "month", "day_of_week"]
-            })
+            human_readable = humanize_crontab(
+                **{
+                    k: cron_parts[k]
+                    for k in ["minute", "hour", "day", "month", "day_of_week"]
+                }
+            )
             return f"Cron: {human_readable} ({crontab})"
         except Exception:
             return f"Cron: {str(trigger)}"

flowerpower/job_queue/base.py

@@ -18,6 +18,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine
 
 from ..cfg import ProjectConfig
 from ..fs import AbstractFileSystem, get_filesystem
+
 # from ..utils.misc import update_config_from_dict
 from ..settings import BACKEND_PROPERTIES
 

flowerpower/job_queue/rq/__init__.py

@@ -1,6 +1,6 @@
+from .manager import RQManager
 from .setup import RQBackend
 from .utils import show_jobs, show_schedules
-from .manager import RQManager
 
 __all__ = [
     "RQManager",

flowerpower/job_queue/rq/manager.py

@@ -11,6 +11,7 @@ import sys
 import time
 import uuid
 from typing import Any, Callable
+
 import duration_parser
 from cron_descriptor import get_description
 from humanize import precisedelta
@@ -658,7 +659,9 @@ class RQManager(BaseJobQueueManager):
         queue = self._queues[queue_name]
         if run_at:
             # Schedule the job to run at a specific time
-            run_at = dt.datetime.fromisoformat(run_at) if isinstance(run_at, str) else run_at
+            run_at = (
+                dt.datetime.fromisoformat(run_at) if isinstance(run_at, str) else run_at
+            )
             job = queue.enqueue_at(
                 run_at,
                 func,
@@ -677,8 +680,14 @@ class RQManager(BaseJobQueueManager):
             )
         elif run_in:
             # Schedule the job to run after a delay
-            run_in = duration_parser.parse(run_in) if isinstance(run_in, str) else run_in
-            run_in = dt.timedelta(seconds=run_in) if isinstance(run_in, (int, float)) else run_in
+            run_in = (
+                duration_parser.parse(run_in) if isinstance(run_in, str) else run_in
+            )
+            run_in = (
+                dt.timedelta(seconds=run_in)
+                if isinstance(run_in, (int, float))
+                else run_in
+            )
             job = queue.enqueue_in(
                 run_in,
                 func,

flowerpower/pipeline/io.py

@@ -40,7 +40,7 @@ class PipelineIOManager:
         name: str,
         src_base_dir: str,
         src_fs: AbstractFileSystem | None = None,
-        src_storage_options: BaseStorageOptions | None = None,
+        src_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ):
         """
@@ -66,7 +66,7 @@ class PipelineIOManager:
             ```
         """
         if src_fs is None:
-            src_fs = get_filesystem(src_base_dir, **(src_storage_options or {}))
+            src_fs = get_filesystem(src_base_dir, storage_options=src_storage_options)
 
         # Use project_cfg attributes for destination paths and filesystem
         dest_pipeline_file = posixpath.join(
@@ -124,7 +124,7 @@ class PipelineIOManager:
         pipelines: dict[str, str] | list[str],
         src_base_dir: str,
         src_fs: AbstractFileSystem | None = None,
-        src_storage_options: BaseStorageOptions | None = None,
+        src_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ):
         """
@@ -173,7 +173,7 @@ class PipelineIOManager:
         self,
         src_base_dir: str,
         src_fs: AbstractFileSystem | None = None,
-        src_storage_options: BaseStorageOptions | None = None,
+        src_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ):
         """Import all pipelines from a given path.
@@ -199,7 +199,7 @@ class PipelineIOManager:
             ```
         """
         if not src_fs:
-            src_fs = get_filesystem(src_base_dir, **(src_storage_options or {}))
+            src_fs = get_filesystem(src_base_dir, storage_options=src_storage_options)
 
         console.print(f"🔍 Search pipelines in [green]{src_base_dir}[/green]...")
 
@@ -241,7 +241,7 @@ class PipelineIOManager:
         name: str,
         dest_base_dir: str,
         dest_fs: AbstractFileSystem | None = None,
-        des_storage_options: BaseStorageOptions | None = None,
+        dest_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ):
         """
@@ -275,7 +275,9 @@ class PipelineIOManager:
             raise ValueError(f"Pipeline {self.project_cfg.name}.{name} does not exist.")
 
         if dest_fs is None:
-            dest_fs = get_filesystem(dest_base_dir, **(des_storage_options or {}))
+            dest_fs = get_filesystem(
+                dest_base_dir, storage_options=dest_storage_options
+            )
 
         # Define destination paths relative to base_dir
         dest_pipeline_file = posixpath.join(
@@ -318,7 +320,7 @@ class PipelineIOManager:
         pipelines: list[str],
         dest_base_dir: str,
         dest_fs: AbstractFileSystem | None = None,
-        dest_storage_options: BaseStorageOptions | None = None,
+        dest_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ):
         """
@@ -361,7 +363,7 @@ class PipelineIOManager:
         self,
         dest_base_dir: str,
         dest_fs: AbstractFileSystem | None = None,
-        dest_storage_options: BaseStorageOptions | None = None,
+        dest_storage_options: dict | BaseStorageOptions | None = {},
         overwrite: bool = False,
     ):
         """Export all pipelines to a given path.

flowerpower/pipeline/job_queue.py

@@ -10,13 +10,14 @@ from uuid import UUID
 from loguru import logger
 from rich import print as rprint
 
+from .. import settings
+
 # Import necessary config types
 from ..cfg import PipelineConfig, ProjectConfig
 from ..fs import AbstractFileSystem
+from ..job_queue import JobQueueManager
 from ..utils.logging import setup_logging
-from ..job_queue import JobQueue
 from .registry import PipelineRegistry
-from .. import settings
 
 setup_logging()
 
@@ -63,7 +64,7 @@ class PipelineJobQueue:
             f"Instantiating worker of type: {self._job_queue_type} for project '{self.project_cfg.name}'"
         )
         # Pass the necessary parts of project_cfg to the Job queue
-        return JobQueue(
+        return JobQueueManager(
             type=self._job_queue_type,
             fs=self._fs,
         )
@@ -116,7 +117,6 @@ class PipelineJobQueue:
         """
         logger.debug(f"Adding immediate job for pipeline: {name}")
 
-       
         pipeline_run_args = {
             # 'name' is not passed to run_func, it's part of the context already in PipelineRunner
             "inputs": inputs,
@@ -332,7 +332,7 @@ class PipelineJobQueue:
 
         # --- Resolve Parameters using pipeline_cfg for defaults ---
         schedule_cfg = pipeline_cfg.schedule
-        #run_cfg = pipeline_cfg.run
+        # run_cfg = pipeline_cfg.run
 
         pipeline_run_args = {
             "inputs": inputs,