FlowerPower 0.11.6.19__py3-none-any.whl → 0.20.0__py3-none-any.whl

flowerpower/cfg/__init__.py

@@ -1,9 +1,9 @@
 from pathlib import Path
 
 import msgspec
+from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
 from munch import Munch
 
-from ..fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
 from .base import BaseConfig
 from .pipeline import PipelineConfig, init_pipeline_config
 from .project import ProjectConfig, init_project_config
@@ -77,7 +77,7 @@ class Config(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(
+            fs = filesystem(
                 base_dir, cached=True, dirfs=True, storage_options=storage_options
             )
         project = ProjectConfig.load(
@@ -123,7 +123,7 @@ class Config(BaseConfig):
             ```
         """
         if fs is None and self.fs is None:
-            self.fs = get_filesystem(
+            self.fs = filesystem(
                 self.base_dir, cached=True, dirfs=True, **storage_options
             )
 

flowerpower/cfg/pipeline/__init__.py

@@ -1,12 +1,14 @@
 import msgspec
 import yaml
+from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
 from hamilton.function_modifiers import source, value
 from munch import Munch, munchify
 
-from ...fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
 from ..base import BaseConfig
 from .adapter import AdapterConfig
+from .run import ExecutorConfig as ExecutorConfig
 from .run import RunConfig
+from .run import WithAdapterConfig as WithAdapterConfig
 from .schedule import ScheduleConfig
 
 
@@ -166,7 +168,7 @@ class PipelineConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(
+            fs = filesystem(
                 base_dir, cached=False, dirfs=True, storage_options=storage_options
             )
         if fs.exists("conf/pipelines"):
@@ -207,7 +209,7 @@ class PipelineConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(
+            fs = filesystem(
                 base_dir, cached=True, dirfs=True, storage_options=storage_options
             )
 

flowerpower/cfg/project/__init__.py

@@ -1,6 +1,6 @@
 import msgspec
+from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
 
-from ...fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
 from ..base import BaseConfig
 from .adapter import AdapterConfig
 from .job_queue import JobQueueConfig
@@ -71,7 +71,7 @@ class ProjectConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(
+            fs = filesystem(
                 base_dir, cached=False, dirfs=True, storage_options=storage_options
             )
         if fs.exists("conf/project.yml"):
@@ -103,7 +103,7 @@ class ProjectConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(
+            fs = filesystem(
                 base_dir, cached=True, dirfs=True, storage_options=storage_options
             )
 

flowerpower/cfg/project/job_queue.py

@@ -1,5 +1,3 @@
-import datetime as dt
-import importlib
 import os
 
 import msgspec
@@ -29,120 +27,6 @@ class JobQueueBackendConfig(BaseConfig):
     verify_ssl: bool = msgspec.field(default=False)
 
 
-class APSDataStoreConfig(JobQueueBackendConfig):
-    type: str = msgspec.field(default=settings.APS_BACKEND_DS or "memory")
-    username: str | None = msgspec.field(default=None)
-    password: str | None = msgspec.field(default=None)
-    host: str | None = msgspec.field(default=None)
-    port: int | None = msgspec.field(default=None)
-    database: str | None = msgspec.field(default=None)
-    schema: str | None = msgspec.field(default=None)
-
-    def __post_init__(self):
-        self.update_settings_from_env()
-        self.host = (
-            settings.APS_BACKEND_DS_HOST
-            or BACKEND_PROPERTIES[self.type]["default_host"]
-        )
-        self.port = (
-            settings.APS_BACKEND_DS_PORT
-            or BACKEND_PROPERTIES[self.type]["default_port"]
-        )
-        self.database = (
-            settings.APS_BACKEND_DS_DB
-            or BACKEND_PROPERTIES[self.type]["default_database"]
-        )
-        self.username = (
-            settings.APS_BACKEND_DS_USERNAME
-            or BACKEND_PROPERTIES[self.type]["default_username"]
-        )
-        self.password = (
-            settings.APS_BACKEND_DS_PASSWORD
-            or BACKEND_PROPERTIES[self.type]["default_password"]
-        )
-
-    def update_settings_from_env(self):
-        if os.getenv("FP_APS_BACKEND_DS") is not None:
-            settings.APS_BACKEND_DS = os.getenv("FP_APS_BACKEND_DS")
-        if os.getenv("FP_APS_BACKEND_DS_USERNAME") is not None:
-            settings.APS_BACKEND_DS_USERNAME = os.getenv("FP_APS_BACKEND_DS_USERNAME")
-        if os.getenv("FP_APS_BACKEND_DS_PASSWORD") is not None:
-            settings.APS_BACKEND_DS_PASSWORD = os.getenv("FP_APS_BACKEND_DS_PASSWORD")
-        if os.getenv("FP_APS_BACKEND_DS_HOST") is not None:
-            settings.APS_BACKEND_DS_HOST = os.getenv("FP_APS_BACKEND_DS_HOST")
-        if os.getenv("FP_APS_BACKEND_DS_PORT") is not None:
-            settings.APS_BACKEND_DS_PORT = int(os.getenv("FP_APS_BACKEND_DS_PORT"))
-        if os.getenv("FP_APS_BACKEND_DS_DB") is not None:
-            settings.APS_BACKEND_DS_DB = os.getenv("FP_APS_BACKEND_DS_DB")
-
-
-class APSEventBrokerConfig(JobQueueBackendConfig):
-    type: str = msgspec.field(default=settings.APS_BACKEND_EB or "memory")
-    username: str | None = msgspec.field(default=None)
-    password: str | None = msgspec.field(default=None)
-    host: str | None = msgspec.field(default=None)
-    port: int | None = msgspec.field(default=None)
-    database: str | None = msgspec.field(default=None)
-    from_ds_sqla: bool = msgspec.field(
-        default_factory=lambda: settings.APS_BACKEND_EB == "postgresql"
-        and settings.APS_BACKEND_DS == "postgresql"
-    )
-
-    def __post_init__(self):
-        self.update_settings_from_env()
-        self.host = (
-            settings.APS_BACKEND_EB_HOST
-            or BACKEND_PROPERTIES[self.type]["default_host"]
-        )
-        self.port = (
-            settings.APS_BACKEND_EB_PORT
-            or BACKEND_PROPERTIES[self.type]["default_port"]
-        )
-        self.database = (
-            settings.APS_BACKEND_EB_DB
-            or BACKEND_PROPERTIES[self.type]["default_database"]
-        )
-        self.username = (
-            settings.APS_BACKEND_EB_USERNAME
-            or BACKEND_PROPERTIES[self.type]["default_username"]
-        )
-        self.password = (
-            settings.APS_BACKEND_EB_PASSWORD
-            or BACKEND_PROPERTIES[self.type]["default_password"]
-        )
-
-    def update_settings_from_env(self):
-        if os.getenv("FP_APS_BACKEND_EB") is not None:
-            settings.APS_BACKEND_EB = os.getenv("FP_APS_BACKEND_EB")
-        if os.getenv("FP_APS_BACKEND_EB_USERNAME") is not None:
-            settings.APS_BACKEND_EB_USERNAME = os.getenv("FP_APS_BACKEND_EB_USERNAME")
-        if os.getenv("FP_APS_BACKEND_EB_PASSWORD") is not None:
-            settings.APS_BACKEND_EB_PASSWORD = os.getenv("FP_APS_BACKEND_EB_PASSWORD")
-        if os.getenv("FP_APS_BACKEND_EB_HOST") is not None:
-            settings.APS_BACKEND_EB_HOST = os.getenv("FP_APS_BACKEND_EB_HOST")
-        if os.getenv("FP_APS_BACKEND_EB_PORT") is not None:
-            settings.APS_BACKEND_EB_PORT = int(os.getenv("FP_APS_BACKEND_EB_PORT"))
-        if os.getenv("FP_APS_BACKEND_EB_DB") is not None:
-            settings.APS_BACKEND_EB_DB = os.getenv("FP_APS_BACKEND_EB_DB")
-
-
-class APSBackendConfig(BaseConfig):
-    data_store: APSDataStoreConfig = msgspec.field(default_factory=APSDataStoreConfig)
-    event_broker: APSEventBrokerConfig = msgspec.field(
-        default_factory=APSEventBrokerConfig
-    )
-    cleanup_interval: int | float | dt.timedelta = msgspec.field(
-        default=settings.APS_CLEANUP_INTERVAL
-    )  # int in seconds
-    max_concurrent_jobs: int = msgspec.field(default=settings.APS_MAX_CONCURRENT_JOBS)
-    default_job_executor: str | None = msgspec.field(default=settings.EXECUTOR)
-    # num_workers: int | None = msgspec.field(default=settings.APS_NUM_WORKERS)
-
-    # def __post_init__(self):
-    #     self.data_store.update_settings_from_env()
-    #     self.event_broker.update_settings_from_env()
-
-
 class RQBackendConfig(JobQueueBackendConfig):
     type: str = msgspec.field(default="redis")
     username: str | None = msgspec.field(default=settings.RQ_BACKEND_USERNAME)
@@ -215,20 +99,9 @@ class JobQueueConfig(BaseConfig):
                     )
                 self.num_workers = self.num_workers or settings.RQ_NUM_WORKERS
 
-            elif self.type == "apscheduler":
-                self.backend = self.backend or APSBackendConfig()
-                if isinstance(self.backend, dict):
-                    self.backend = APSBackendConfig.from_dict(self.backend)
-                elif isinstance(self.backend, APSBackendConfig):
-                    pass
-                else:
-                    raise ValueError(
-                        f"Invalid backend type for APScheduler: {type(self.backend)}"
-                    )
-                self.num_workers = self.num_workers or settings.APS_NUM_WORKERS
             else:
                 raise ValueError(
-                    f"Invalid job queue type: {self.type}. Valid types: {['rq', 'apscheduler', 'huey']}"
+                    f"Invalid job queue type: {self.type}. Valid types: ['rq']"
                 )
 
     def update_type(self, type: str):

flowerpower/cli/__init__.py

@@ -18,7 +18,7 @@ app.add_typer(
     pipeline_app, name="pipeline", help="Manage and execute FlowerPower pipelines"
 )
 
-if importlib.util.find_spec("apscheduler") or importlib.util.find_spec("rq"):
+if importlib.util.find_spec("rq"):
     from .job_queue import app as job_queue_app
 
     app.add_typer(
@@ -53,7 +53,7 @@ def init(
         "rq",
         "--job-queue-type",
         "-q",
-        help="Job queue backend type to use (rq, apscheduler)",
+        help="Job queue backend type to use (rq)",
     ),
 ):
     """
@@ -69,7 +69,7 @@ def init(
         base_dir: Base directory where the project will be created. If not provided,
                   the current directory's parent will be used
         storage_options: Storage options for filesystem access, as a JSON or dict string
-        job_queue_type: Type of job queue backend to use (rq, apscheduler)
+        job_queue_type: Type of job queue backend to use (rq)
 
     Examples:
         # Create a project in the current directory using its name
@@ -81,8 +81,8 @@ def init(
         # Create a project in a specific location
         $ flowerpower init --name my-project --base-dir /path/to/projects
 
-        # Create a project with APScheduler as the job queue backend
-        $ flowerpower init --job-queue-type apscheduler
+        # Create a project with RQ as the job queue backend (default)
+        $ flowerpower init --job-queue-type rq
     """
     parsed_storage_options = {}
     if storage_options:

flowerpower/cli/cfg.py

@@ -1,8 +1,5 @@
 import typer
 
-from ..cfg import Config
-from ..cli.utils import parse_dict_or_list_param
-
 app = typer.Typer(help="Config management commands")