FlowerPower 0.9.12.4__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

flowerpower/cfg/project/adapter.py

@@ -0,0 +1,57 @@
+import msgspec
+from munch import munchify
+
+from ... import settings
+from ..base import BaseConfig
+
+
+class HamiltonTrackerConfig(BaseConfig):
+    username: str | None = msgspec.field(default=None)
+    api_url: str = msgspec.field(default=settings.HAMILTON_API_URL)
+    ui_url: str = msgspec.field(default=settings.HAMILTON_UI_URL)
+    api_key: str | None = msgspec.field(default=None)
+    verify: bool = msgspec.field(default=False)
+
+
+class MLFlowConfig(BaseConfig):
+    tracking_uri: str | None = msgspec.field(default=None)
+    registry_uri: str | None = msgspec.field(default=None)
+    artifact_location: str | None = msgspec.field(default=None)
+
+
+class OpenTelemetryConfig(BaseConfig):
+    host: str = msgspec.field(default="localhost")
+    port: int = msgspec.field(default=6831)
+
+
+# class OpenLineageConfig(BaseConfig):
+#     from openlineage.client import OpenLineageClientOptions
+#     from openlineage.client.transport import Transport
+#     from openlineage.client.transport import TransportFactory
+#     url: str | None = msgspec.field(default=None)
+#     options: OpenLineageClientOptions | None = msgspec.field(
+#         default=None)
+#     transport: Transport | None = msgspec.field(default=None)
+#     factory: TransportFactory | None = msgspec.field(
+#         default=None)
+#     config: dict | None = msgspec.field(default=None)
+
+
+class RayConfig(BaseConfig):
+    ray_init_config: dict | None = msgspec.field(default=None)
+    shutdown_ray_on_completion: bool = msgspec.field(default=False)
+
+    def __post_init__(self):
+        if isinstance(self.ray_init_config, dict):
+            self.ray_init_config = munchify(self.ray_init_config)
+
+
+class AdapterConfig(BaseConfig):
+    hamilton_tracker: HamiltonTrackerConfig = msgspec.field(
+        default_factory=HamiltonTrackerConfig
+    )
+    mlflow: MLFlowConfig = msgspec.field(default_factory=MLFlowConfig)
+    ray: RayConfig = msgspec.field(default_factory=RayConfig)
+    opentelemetry: OpenTelemetryConfig = msgspec.field(
+        default_factory=OpenTelemetryConfig
+    )

flowerpower/cfg/project/job_queue.py

@@ -0,0 +1,165 @@
+import datetime as dt
+
+import msgspec
+
+from ... import settings
+from ..base import BaseConfig
+
+
+
+class JobQueueBackendConfig(BaseConfig):
+    """
+    Job Queue backend configuration for FlowerPower.
+    Inherits from BaseConfig and adapts Redis logic.
+    """
+
+    type: str | None = msgspec.field(default=None)
+    uri: str | None = msgspec.field(default=None)
+    username: str | None = msgspec.field(default=None)
+    password: str | None = msgspec.field(default=None)
+    host: str | None = msgspec.field(default=None)
+    port: int | None = msgspec.field(default=None)
+    database: int | None = msgspec.field(default=None)
+    ssl: bool = msgspec.field(default=False)
+    cert_file: str | None = msgspec.field(default=None)
+    key_file: str | None = msgspec.field(default=None)
+    ca_file: str | None = msgspec.field(default=None)
+    verify_ssl: bool = msgspec.field(default=False)
+
+
+class APSDataStoreConfig(JobQueueBackendConfig):
+    type: str = msgspec.field(default=settings.APS_BACKEND_DS)
+    host: str = msgspec.field(
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_DS][
+            "default_host"
+        ]
+    )
+    port: int = msgspec.field(
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_DS][
+            "default_port"
+        ]
+    )
+    schema: str | None = msgspec.field(default=settings.APS_SCHEMA_DS)
+    username: str = msgspec.field(
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_DS][
+            "default_username"
+        ]
+    )
+
+
+class APSEventBrokerConfig(JobQueueBackendConfig):
+    type: str = msgspec.field(default=settings.APS_BACKEND_EB)
+    host: str = msgspec.field(
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_EB][
+            "default_host"
+        ]
+    )
+    port: int = msgspec.field(
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_EB][
+            "default_port"
+        ]
+    )
+    username: str = msgspec.field(
+        default=settings.BACKEND_PROPERTIES[settings.APS_BACKEND_EB][
+            "default_username"
+        ]
+    )
+    from_ds_sqla: bool = msgspec.field(
+        default_factory=lambda: settings.APS_BACKEND_EB == "postgresql"
+        and settings.APS_BACKEND_DS == "postgresql"
+    )
+
+
+class APSBackendConfig(BaseConfig):
+    data_store: APSDataStoreConfig = msgspec.field(default_factory=APSDataStoreConfig)
+    event_broker: APSEventBrokerConfig = msgspec.field(
+        default_factory=APSEventBrokerConfig
+    )
+    cleanup_interval: int | float | dt.timedelta = msgspec.field(
+        default=settings.APS_CLEANUP_INTERVAL
+    )  # int in secods
+    max_concurrent_jobs: int = msgspec.field(
+        default=settings.APS_MAX_CONCURRENT_JOBS
+    )
+    default_job_executor: str | None = msgspec.field(default=settings.EXECUTOR)
+    num_workers: int | None = msgspec.field(default=settings.APS_NUM_WORKERS)
+
+
+class RQBackendConfig(JobQueueBackendConfig):
+    type: str = msgspec.field(default="redis")
+    host: str = msgspec.field(
+        default=settings.BACKEND_PROPERTIES["redis"]["default_host"]
+    )
+    port: int = msgspec.field(
+        default=settings.BACKEND_PROPERTIES["redis"]["default_port"]
+    )
+    database: int = msgspec.field(
+        default=settings.BACKEND_PROPERTIES["redis"]["default_database"]
+    )
+    queues: str | list[str] = msgspec.field(
+        default_factory=lambda: settings.RQ_QUEUES
+    )
+    num_workers: int = msgspec.field(
+        default=settings.RQ_NUM_WORKERS
+    )  # int in secods
+
+
+class HueyBackendConfig(JobQueueBackendConfig):
+    pass
+
+
+class JobQueueConfig(BaseConfig):
+    type: str | None = msgspec.field(default="rq")
+    backend: dict | None = msgspec.field(default=None)
+
+    def __post_init__(self):
+        if self.type is not None:
+            self.type = self.type.lower()
+            if self.type == "rq":
+                if self.backend is None:
+                    self.backend = RQBackendConfig()
+                else:
+                    if isinstance(self.backend, dict):
+                        self.backend = RQBackendConfig.from_dict(self.backend)
+                    elif isinstance(self.backend, RQBackendConfig):
+                        pass
+                    else:
+                        raise ValueError(
+                            f"Invalid backend type for RQ: {type(self.backend)}"
+                        )
+            elif self.type == "apscheduler":
+                if self.backend is None:
+                    self.backend = APSBackendConfig( )
+                else:
+                    if isinstance(self.backend, dict):
+                        self.backend = APSBackendConfig.from_dict(self.backend)
+                    elif isinstance(self.backend, APSBackendConfig):
+                        pass
+                    else:
+                        raise ValueError(
+                            f"Invalid backend type for APScheduler: {type(self.backend)}"
+                        )
+
+            elif self.type == "huey":
+                if self.backend is None:
+                    self.backend = HueyBackendConfig()
+                else:
+                    if isinstance(self.backend, dict):
+                        self.backend = HueyBackendConfig.from_dict(self.backend)
+                    elif isinstance(self.backend, HueyBackendConfig):
+                        pass
+                    else:
+                        raise ValueError(
+                            f"Invalid backend type for Huey: {type(self.backend)}"
+                        )
+                    self.backend = HueyBackendConfig(**self.backend)
+            else:
+                raise ValueError(
+                    f"Invalid job queue type: {self.type}. Valid types: {['rq', 'apscheduler', 'huey']}"
+                )
+
+    def update_type(self, type: str):
+        if type != self.type:
+            self.type = type
+            self.backend = None
+            self.__post_init__()

flowerpower/cli/__init__.py

@@ -1,69 +1,126 @@
 import importlib
+import os
 
 import typer
 from loguru import logger
 
 from ..flowerpower import init as init_
 from .pipeline import app as pipeline_app
+from .utils import parse_dict_or_list_param
 
-app = typer.Typer()
+app = typer.Typer(
+    help="FlowerPower: A framework for building, executing, and managing data processing pipelines",
+    rich_markup_mode="rich",
+)
 
 
-app.add_typer(pipeline_app, name="pipeline")
+app.add_typer(pipeline_app, name="pipeline", help="Manage and execute FlowerPower pipelines")
 
-if importlib.util.find_spec("apscheduler"):
-    from .scheduler import app as scheduler_app
+if importlib.util.find_spec("apscheduler") or importlib.util.find_spec("rq"):
+    from .job_queue import app as job_queue_app
 
-    app.add_typer(scheduler_app, name="scheduler")
+    app.add_typer(job_queue_app, name="job-queue", help="Manage job queue workers and scheduled tasks")
 
 if importlib.util.find_spec("paho"):
     from .mqtt import app as mqtt_app
 
-    app.add_typer(mqtt_app, name="mqtt")
+    app.add_typer(mqtt_app, name="mqtt", help="Connect pipelines to MQTT message brokers")
 
 
 @app.command()
 def init(
-    project_name: str = None,
-    base_dir: str = None,
-    storage_options: str = None,
+    project_name: str = typer.Option(None, "--name", "-n", help="Name of the FlowerPower project to create"),
+    base_dir: str = typer.Option(None, "--base-dir", "-d", help="Base directory where the project will be created"),
+    storage_options: str = typer.Option(None, "--storage-options", "-s", help="Storage options as a JSON or dict string"),
+    job_queue_type: str = typer.Option("rq", "--job-queue-type", "-q", help="Job queue backend type to use (rq, apscheduler)"),
 ):
     """
-    Initialize the FlowerPower application.
-
+    Initialize a new FlowerPower project.
+    
+    This command creates a new FlowerPower project with the necessary directory structure
+    and configuration files. If no project name is provided, the current directory name
+    will be used as the project name.
+    
     Args:
-        name (str): The name of the application.
-        base_dir (str, optional): The base path of the application. Defaults to "".
-        storage_options (str, optional): The filesystem storage options for the task. Defaults to None".
+        project_name: Name of the FlowerPower project to create. If not provided, 
+                      the current directory name will be used
+        base_dir: Base directory where the project will be created. If not provided,
+                  the current directory's parent will be used
+        storage_options: Storage options for filesystem access, as a JSON or dict string
+        job_queue_type: Type of job queue backend to use (rq, apscheduler)
+    
+    Examples:
+        # Create a project in the current directory using its name
+        $ flowerpower init
+        
+        # Create a project with a specific name
+        $ flowerpower init --name my-awesome-project
+        
+        # Create a project in a specific location
+        $ flowerpower init --name my-project --base-dir /path/to/projects
+        
+        # Create a project with APScheduler as the job queue backend
+        $ flowerpower init --job-queue-type apscheduler
     """
-    storage_options = eval(storage_options) if storage_options is not None else {}
+    parsed_storage_options = {}
+    if storage_options:
+        try:
+            parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
+        except Exception as e:
+            logger.error(f"Error parsing storage options: {e}")
+            raise typer.Exit(code=1)
 
-    init_(
-        name=project_name,
-        base_dir=base_dir,
-        storage_options=storage_options,
-    )
+    try:
+        init_(
+            name=project_name,
+            base_dir=base_dir,
+            storage_options=parsed_storage_options,
+            job_queue_type=job_queue_type,
+        )
+    except Exception as e:
+        logger.error(f"Error initializing project: {e}")
+        raise typer.Exit(code=1)
 
 
 @app.command()
 def ui(
-    port: int = 8241,
-    base_dir: str = "~/.hamilton/db",
-    no_migration: bool = False,
-    no_open: bool = False,
-    settings_file: str = "mini",
-    config_file: str = None,
+    port: int = typer.Option(8241, "--port", "-p", help="Port to run the UI server on"),
+    base_dir: str = typer.Option("~/.hamilton/db", "--base-dir", "-d", help="Base directory for Hamilton UI data"),
+    no_migration: bool = typer.Option(False, "--no-migration", help="Skip running database migrations"),
+    no_open: bool = typer.Option(False, "--no-open", help="Don't automatically open the UI in a browser"),
+    settings_file: str = typer.Option("mini", "--settings", "-s", help="Settings file to use for the UI"),
+    config_file: str = typer.Option(None, "--config", "-c", help="Configuration file to use for the UI"),
 ):
     """
-    Start the Hamilton UI.
-
+    Start the Hamilton UI web application.
+    
+    This command launches the Hamilton UI, which provides a web interface for 
+    visualizing and interacting with your FlowerPower pipelines. The UI allows you
+    to explore pipeline execution graphs, view results, and manage jobs.
+    
     Args:
-        port (int, optional): The port to run the UI on. Defaults to 8241.
-        base_dir (str, optional): The base path for the UI. Defaults to "~/.hamilton/db".
-        no_migration (bool, optional): Whether to run the migration. Defaults to False.
-        no_open (bool, optional): Whether to open the UI in the browser. Defaults to False.
-        settings_file (str, optional): The settings file to use. Defaults to "mini".
-        config_file (str, optional): The config file to use. Defaults to None.
+        port: Port to run the UI server on
+        base_dir: Base directory where the UI will store its data
+        no_migration: Skip running database migrations on startup
+        no_open: Prevent automatically opening the browser
+        settings_file: Settings profile to use (mini, dev, prod)
+        config_file: Optional custom configuration file path
+    
+    Examples:
+        # Start the UI with default settings
+        $ flowerpower ui
+        
+        # Run the UI on a specific port
+        $ flowerpower ui --port 9000
+        
+        # Use a custom data directory
+        $ flowerpower ui --base-dir ~/my-project/.hamilton-data
+        
+        # Start without opening a browser
+        $ flowerpower ui --no-open
+        
+        # Use production settings
+        $ flowerpower ui --settings prod
     """
     try:
         from hamilton_ui import commands
@@ -76,7 +133,7 @@ def ui(
 
     commands.run(
         port=port,
-        base_dir=base_dir,
+        base_dir=os.path.expanduser(base_dir),
         no_migration=no_migration,
         no_open=no_open,
         settings_file=settings_file,