FlowerPower 1.0.0b1__py3-none-any.whl → 1.0.0b3__py3-none-any.whl

flowerpower/__init__.py

@@ -1,10 +1,9 @@
 import importlib.metadata
 
+from .cfg import Config, PipelineConfig, ProjectConfig
 from .flowerpower import init as init_project  # noqa: E402
-
-from .pipeline import PipelineManager
 from .job_queue import JobQueueManager
-from .cfg import Config, ProjectConfig, PipelineConfig
+from .pipeline import PipelineManager
 
 __version__ = importlib.metadata.version("FlowerPower")
 

flowerpower/cfg/__init__.py

@@ -3,7 +3,7 @@ from pathlib import Path
 import msgspec
 from munch import Munch
 
-from ..fs import AbstractFileSystem, get_filesystem
+from ..fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
 from .base import BaseConfig
 from .pipeline import PipelineConfig, init_pipeline_config
 from .project import ProjectConfig, init_project_config
@@ -51,7 +51,7 @@ class Config(BaseConfig):
         pipeline_name: str | None = None,
         job_queue_type: str | None = None,
         fs: AbstractFileSystem | None = None,
-        storage_options: dict | Munch = Munch(),
+        storage_options: dict | BaseStorageOptions | None = {},
     ):
         """Load both project and pipeline configurations.
 
@@ -77,7 +77,9 @@ class Config(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(base_dir, cached=True, dirfs=True, **storage_options)
+            fs = get_filesystem(
+                base_dir, cached=True, dirfs=True, storage_options=storage_options
+            )
         project = ProjectConfig.load(
             base_dir=base_dir,
             name=name,
@@ -105,7 +107,7 @@ class Config(BaseConfig):
         project: bool = False,
         pipeline: bool = True,
         fs: AbstractFileSystem | None = None,
-        storage_options: dict | Munch = Munch(),
+        storage_options: dict | BaseStorageOptions | None = {},
     ):
         """Save project and/or pipeline configurations.
 
@@ -144,7 +146,7 @@ def load(
     base_dir: str,
     name: str | None = None,
     pipeline_name: str | None = None,
-    storage_options: dict | Munch = Munch(),
+    storage_options: dict | BaseStorageOptions | None = {},
     fs: AbstractFileSystem | None = None,
 ):
     """Helper function to load configuration.
@@ -180,7 +182,7 @@ def save(
     project: bool = False,
     pipeline: bool = True,
     fs: AbstractFileSystem | None = None,
-    storage_options: dict | Munch = Munch(),
+    storage_options: dict | BaseStorageOptions | None = {},
 ):
     """Helper function to save configuration.
 
@@ -210,7 +212,7 @@ def init_config(
     pipeline_name: str | None = None,
     job_queue_type: str | None = None,
     fs: AbstractFileSystem | None = None,
-    storage_options: dict | Munch = Munch(),
+    storage_options: dict | BaseStorageOptions | None = {},
 ):
     """Initialize a new configuration with both project and pipeline settings.
 
@@ -251,4 +253,4 @@ def init_config(
         fs=fs,
         storage_options=storage_options,
     )
-    return Config(pipeline=pipeline_cfg, project=project_cfg, fs=fs, base_dir=base_dir)
+    return Config(pipeline=pipeline_cfg, project=project_cfg, fs=fs, base_dir=base_dir)

flowerpower/cfg/pipeline/__init__.py

@@ -3,7 +3,7 @@ import yaml
 from hamilton.function_modifiers import source, value
 from munch import Munch, munchify
 
-from ...fs import AbstractFileSystem, get_filesystem
+from ...fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
 from ..base import BaseConfig
 from .adapter import AdapterConfig
 from .run import RunConfig
@@ -144,7 +144,7 @@ class PipelineConfig(BaseConfig):
         base_dir: str = ".",
         name: str | None = None,
         fs: AbstractFileSystem | None = None,
-        storage_options: dict | Munch = Munch(),
+        storage_options: dict | BaseStorageOptions | None = {},
     ):
         """Load pipeline configuration from a YAML file.
 
@@ -166,7 +166,9 @@ class PipelineConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(base_dir, cached=False, dirfs=True, **storage_options)
+            fs = get_filesystem(
+                base_dir, cached=False, dirfs=True, storage_options=storage_options
+            )
         if fs.exists("conf/pipelines"):
             if name is not None:
                 pipeline = PipelineConfig.from_yaml(
@@ -186,7 +188,7 @@ class PipelineConfig(BaseConfig):
         name: str | None = None,
         base_dir: str = ".",
         fs: AbstractFileSystem | None = None,
-        storage_options: dict | Munch = Munch(),
+        storage_options: dict | BaseStorageOptions | None = {},
     ):
         """Save pipeline configuration to a YAML file.
 
@@ -205,7 +207,9 @@ class PipelineConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(base_dir, cached=True, dirfs=True, **storage_options)
+            fs = get_filesystem(
+                base_dir, cached=True, dirfs=True, storage_options=storage_options
+            )
 
         fs.makedirs("conf/pipelines", exist_ok=True)
         if name is not None:
@@ -224,7 +228,7 @@ def init_pipeline_config(
     base_dir: str = ".",
     name: str | None = None,
     fs: AbstractFileSystem | None = None,
-    storage_options: dict | Munch = Munch(),
+    storage_options: dict | BaseStorageOptions | None = {},
 ):
     """Initialize a new pipeline configuration.
 
@@ -251,4 +255,4 @@ def init_pipeline_config(
         base_dir=base_dir, name=name, fs=fs, storage_options=storage_options
     )
     pipeline.save(name=name, base_dir=base_dir, fs=fs, storage_options=storage_options)
-    return pipeline
+    return pipeline

flowerpower/cfg/project/__init__.py

@@ -1,7 +1,6 @@
 import msgspec
-from munch import Munch
 
-from ...fs import AbstractFileSystem, get_filesystem
+from ...fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
 from ..base import BaseConfig
 from .adapter import AdapterConfig
 from .job_queue import JobQueueConfig
@@ -48,7 +47,7 @@ class ProjectConfig(BaseConfig):
         name: str | None = None,
         job_queue_type: str | None = None,
         fs: AbstractFileSystem | None = None,
-        storage_options: dict | Munch = Munch(),
+        storage_options: dict | BaseStorageOptions | None = {},
     ):
         """Load project configuration from a YAML file.
 
@@ -72,7 +71,9 @@ class ProjectConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(base_dir, cached=False, dirfs=True, **storage_options)
+            fs = get_filesystem(
+                base_dir, cached=False, dirfs=True, storage_options=storage_options
+            )
         if fs.exists("conf/project.yml"):
             project = ProjectConfig.from_yaml(path="conf/project.yml", fs=fs)
         else:
@@ -87,7 +88,7 @@ class ProjectConfig(BaseConfig):
         self,
         base_dir: str = ".",
         fs: AbstractFileSystem | None = None,
-        storage_options: dict | Munch = Munch(),
+        storage_options: dict | BaseStorageOptions | None = {},
     ):
         """Save project configuration to a YAML file.
 
@@ -102,7 +103,9 @@ class ProjectConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(base_dir, cached=True, dirfs=True, **storage_options)
+            fs = get_filesystem(
+                base_dir, cached=True, dirfs=True, storage_options=storage_options
+            )
 
         fs.makedirs("conf", exist_ok=True)
         self.to_yaml(path="conf/project.yml", fs=fs)
@@ -113,7 +116,7 @@ def init_project_config(
     name: str | None = None,
     job_queue_type: str | None = None,
     fs: AbstractFileSystem | None = None,
-    storage_options: dict | Munch = Munch(),
+    storage_options: dict | BaseStorageOptions | None = {},
 ):
     """Initialize a new project configuration.
 
@@ -146,4 +149,4 @@ def init_project_config(
         storage_options=storage_options,
     )
     project.save(base_dir=base_dir, fs=fs, storage_options=storage_options)
-    return project
+    return project

flowerpower/cfg/project/job_queue.py

@@ -6,7 +6,6 @@ from ... import settings
 from ..base import BaseConfig
 
 
-
 class JobQueueBackendConfig(BaseConfig):
     """
     Job Queue backend configuration for FlowerPower.
@@ -30,39 +29,27 @@ class JobQueueBackendConfig(BaseConfig):
 class APSDataStoreConfig(JobQueueBackendConfig):
     type: str = msgspec.field(default=settings.APS_BACKEND_DS)
     host: str = msgspec.field(
-        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_DS][
-            "default_host"
-        ]
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_DS]["default_host"]
     )
     port: int = msgspec.field(
-        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_DS][
-            "default_port"
-        ]
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_DS]["default_port"]
     )
     schema: str | None = msgspec.field(default=settings.APS_SCHEMA_DS)
     username: str = msgspec.field(
-        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_DS][
-            "default_username"
-        ]
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_DS]["default_username"]
     )
 
 
 class APSEventBrokerConfig(JobQueueBackendConfig):
     type: str = msgspec.field(default=settings.APS_BACKEND_EB)
     host: str = msgspec.field(
-        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_EB][
-            "default_host"
-        ]
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_EB]["default_host"]
     )
     port: int = msgspec.field(
-        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_EB][
-            "default_port"
-        ]
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_EB]["default_port"]
     )
     username: str = msgspec.field(
-        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_EB][
-            "default_username"
-        ]
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_EB]["default_username"]
     )
     from_ds_sqla: bool = msgspec.field(
         default_factory=lambda: settings.APS_BACKEND_EB == "postgresql"
@@ -78,9 +65,7 @@ class APSBackendConfig(BaseConfig):
     cleanup_interval: int | float | dt.timedelta = msgspec.field(
         default=settings.APS_CLEANUP_INTERVAL
     )  # int in secods
-    max_concurrent_jobs: int = msgspec.field(
-        default=settings.APS_MAX_CONCURRENT_JOBS
-    )
+    max_concurrent_jobs: int = msgspec.field(default=settings.APS_MAX_CONCURRENT_JOBS)
     default_job_executor: str | None = msgspec.field(default=settings.EXECUTOR)
     num_workers: int | None = msgspec.field(default=settings.APS_NUM_WORKERS)
 
@@ -96,12 +81,8 @@ class RQBackendConfig(JobQueueBackendConfig):
     database: int = msgspec.field(
         default=settings.BACKEND_PROPERTIES["redis"]["default_database"]
     )
-    queues: str | list[str] = msgspec.field(
-        default_factory=lambda: settings.RQ_QUEUES
-    )
-    num_workers: int = msgspec.field(
-        default=settings.RQ_NUM_WORKERS
-    )  # int in secods
+    queues: str | list[str] = msgspec.field(default_factory=lambda: settings.RQ_QUEUES)
+    num_workers: int = msgspec.field(default=settings.RQ_NUM_WORKERS)  # int in secods
 
 
 class HueyBackendConfig(JobQueueBackendConfig):
@@ -129,7 +110,7 @@ class JobQueueConfig(BaseConfig):
                         )
             elif self.type == "apscheduler":
                 if self.backend is None:
-                    self.backend = APSBackendConfig( )
+                    self.backend = APSBackendConfig()
                 else:
                     if isinstance(self.backend, dict):
                         self.backend = APSBackendConfig.from_dict(self.backend)

flowerpower/cli/__init__.py

@@ -14,58 +14,82 @@ app = typer.Typer(
 )
 
 
-app.add_typer(pipeline_app, name="pipeline", help="Manage and execute FlowerPower pipelines")
+app.add_typer(
+    pipeline_app, name="pipeline", help="Manage and execute FlowerPower pipelines"
+)
 
 if importlib.util.find_spec("apscheduler") or importlib.util.find_spec("rq"):
     from .job_queue import app as job_queue_app
 
-    app.add_typer(job_queue_app, name="job-queue", help="Manage job queue workers and scheduled tasks")
+    app.add_typer(
+        job_queue_app,
+        name="job-queue",
+        help="Manage job queue workers and scheduled tasks",
+    )
 
 if importlib.util.find_spec("paho"):
     from .mqtt import app as mqtt_app
 
-    app.add_typer(mqtt_app, name="mqtt", help="Connect pipelines to MQTT message brokers")
+    app.add_typer(
+        mqtt_app, name="mqtt", help="Connect pipelines to MQTT message brokers"
+    )
 
 
 @app.command()
 def init(
-    project_name: str = typer.Option(None, "--name", "-n", help="Name of the FlowerPower project to create"),
-    base_dir: str = typer.Option(None, "--base-dir", "-d", help="Base directory where the project will be created"),
-    storage_options: str = typer.Option(None, "--storage-options", "-s", help="Storage options as a JSON or dict string"),
-    job_queue_type: str = typer.Option("rq", "--job-queue-type", "-q", help="Job queue backend type to use (rq, apscheduler)"),
+    project_name: str = typer.Option(
+        None, "--name", "-n", help="Name of the FlowerPower project to create"
+    ),
+    base_dir: str = typer.Option(
+        None,
+        "--base-dir",
+        "-d",
+        help="Base directory where the project will be created",
+    ),
+    storage_options: str = typer.Option(
+        None, "--storage-options", "-s", help="Storage options as a JSON or dict string"
+    ),
+    job_queue_type: str = typer.Option(
+        "rq",
+        "--job-queue-type",
+        "-q",
+        help="Job queue backend type to use (rq, apscheduler)",
+    ),
 ):
     """
     Initialize a new FlowerPower project.
-    
+
     This command creates a new FlowerPower project with the necessary directory structure
     and configuration files. If no project name is provided, the current directory name
     will be used as the project name.
-    
+
     Args:
-        project_name: Name of the FlowerPower project to create. If not provided, 
+        project_name: Name of the FlowerPower project to create. If not provided,
                       the current directory name will be used
         base_dir: Base directory where the project will be created. If not provided,
                   the current directory's parent will be used
         storage_options: Storage options for filesystem access, as a JSON or dict string
         job_queue_type: Type of job queue backend to use (rq, apscheduler)
-    
+
     Examples:
         # Create a project in the current directory using its name
         $ flowerpower init
-        
+
         # Create a project with a specific name
         $ flowerpower init --name my-awesome-project
-        
+
         # Create a project in a specific location
         $ flowerpower init --name my-project --base-dir /path/to/projects
-        
+
         # Create a project with APScheduler as the job queue backend
         $ flowerpower init --job-queue-type apscheduler
     """
     parsed_storage_options = {}
     if storage_options:
         try:
-            parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
+            parsed_storage_options = (
+                parse_dict_or_list_param(storage_options, "dict") or {}
+            )
         except Exception as e:
             logger.error(f"Error parsing storage options: {e}")
             raise typer.Exit(code=1)
@@ -85,19 +109,29 @@ def init(
 @app.command()
 def ui(
     port: int = typer.Option(8241, "--port", "-p", help="Port to run the UI server on"),
-    base_dir: str = typer.Option("~/.hamilton/db", "--base-dir", "-d", help="Base directory for Hamilton UI data"),
-    no_migration: bool = typer.Option(False, "--no-migration", help="Skip running database migrations"),
-    no_open: bool = typer.Option(False, "--no-open", help="Don't automatically open the UI in a browser"),
-    settings_file: str = typer.Option("mini", "--settings", "-s", help="Settings file to use for the UI"),
-    config_file: str = typer.Option(None, "--config", "-c", help="Configuration file to use for the UI"),
+    base_dir: str = typer.Option(
+        "~/.hamilton/db", "--base-dir", "-d", help="Base directory for Hamilton UI data"
+    ),
+    no_migration: bool = typer.Option(
+        False, "--no-migration", help="Skip running database migrations"
+    ),
+    no_open: bool = typer.Option(
+        False, "--no-open", help="Don't automatically open the UI in a browser"
+    ),
+    settings_file: str = typer.Option(
+        "mini", "--settings", "-s", help="Settings file to use for the UI"
+    ),
+    config_file: str = typer.Option(
+        None, "--config", "-c", help="Configuration file to use for the UI"
+    ),
 ):
     """
     Start the Hamilton UI web application.
-    
-    This command launches the Hamilton UI, which provides a web interface for 
+
+    This command launches the Hamilton UI, which provides a web interface for
     visualizing and interacting with your FlowerPower pipelines. The UI allows you
     to explore pipeline execution graphs, view results, and manage jobs.
-    
+
     Args:
         port: Port to run the UI server on
         base_dir: Base directory where the UI will store its data
@@ -105,20 +139,20 @@ def ui(
         no_open: Prevent automatically opening the browser
         settings_file: Settings profile to use (mini, dev, prod)
         config_file: Optional custom configuration file path
-    
+
     Examples:
         # Start the UI with default settings
         $ flowerpower ui
-        
+
         # Run the UI on a specific port
         $ flowerpower ui --port 9000
-        
+
         # Use a custom data directory
         $ flowerpower ui --base-dir ~/my-project/.hamilton-data
-        
+
         # Start without opening a browser
         $ flowerpower ui --no-open
-        
+
         # Use production settings
         $ flowerpower ui --settings prod
     """