FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl

flowerpower/__init__.py

@@ -1,22 +1,18 @@
 import importlib.metadata
 
 from .cfg import Config, PipelineConfig, ProjectConfig
-from .flowerpower import FlowerPower, FlowerPowerProject
-from .flowerpower import init as init_project  # noqa: E402
-from .job_queue import JobQueueManager  # noqa: E402
+from .flowerpower import FlowerPower, FlowerPowerProject, create_project
 from .pipeline import PipelineManager
 
 __version__ = importlib.metadata.version("FlowerPower")
 
 __all__ = [
     "__version__",
-    "init_project",
+    "create_project",
     "FlowerPower",
     "FlowerPowerProject",
     "PipelineManager",
-    "JobQueueManager",
     "Config",
     "ProjectConfig",
     "PipelineConfig",
-    "PipelineConfig",
 ]

flowerpower/cfg/__init__.py

@@ -1,9 +1,10 @@
 from pathlib import Path
 
 import msgspec
+from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
 from munch import Munch
 
-from ..fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
+from ..settings import CONFIG_DIR, PIPELINES_DIR
 from .base import BaseConfig
 from .pipeline import PipelineConfig, init_pipeline_config
 from .project import ProjectConfig, init_project_config
@@ -49,7 +50,6 @@ class Config(BaseConfig):
         base_dir: str = ".",
         name: str | None = None,
         pipeline_name: str | None = None,
-        job_queue_type: str | None = None,
         fs: AbstractFileSystem | None = None,
         storage_options: dict | BaseStorageOptions | None = {},
     ):
@@ -59,7 +59,6 @@ class Config(BaseConfig):
             base_dir (str, optional): Base directory for configurations. Defaults to ".".
             name (str | None, optional): Project name. Defaults to None.
             pipeline_name (str | None, optional): Pipeline name. Defaults to None.
-            job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
             fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
             storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
 
@@ -72,18 +71,16 @@ class Config(BaseConfig):
                 base_dir="my_project",
                 name="test_project",
                 pipeline_name="etl",
-                job_queue_type="rq"
             )
             ```
         """
         if fs is None:
-            fs = get_filesystem(
+            fs = filesystem(
                 base_dir, cached=True, dirfs=True, storage_options=storage_options
             )
         project = ProjectConfig.load(
             base_dir=base_dir,
             name=name,
-            job_queue_type=job_queue_type,
             fs=fs,
             storage_options=storage_options,
         )
@@ -123,15 +120,15 @@ class Config(BaseConfig):
             ```
         """
         if fs is None and self.fs is None:
-            self.fs = get_filesystem(
+            self.fs = filesystem(
                 self.base_dir, cached=True, dirfs=True, **storage_options
             )
 
-        if not self.fs.exists("conf"):
-            self.fs.makedirs("conf")
+        if not self.fs.exists(CONFIG_DIR):
+            self.fs.makedirs(CONFIG_DIR)
 
         if pipeline:
-            self.fs.makedirs("conf/pipelines", exist_ok=True)
+            self.fs.makedirs(PIPELINES_DIR, exist_ok=True)
             h_params = self.pipeline.pop("h_params") if self.pipeline.h_params else None
             self.pipeline.to_yaml(
                 path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
@@ -210,7 +207,6 @@ def init_config(
     base_dir: str = ".",
     name: str | None = None,
     pipeline_name: str | None = None,
-    job_queue_type: str | None = None,
     fs: AbstractFileSystem | None = None,
     storage_options: dict | BaseStorageOptions | None = {},
 ):
@@ -223,7 +219,6 @@ def init_config(
         base_dir (str, optional): Base directory for configurations. Defaults to ".".
         name (str | None, optional): Project name. Defaults to None.
         pipeline_name (str | None, optional): Pipeline name. Defaults to None.
-        job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
         fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
         storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
 
@@ -236,7 +231,6 @@ def init_config(
             base_dir="my_project",
             name="test_project",
             pipeline_name="data-pipeline",
-            job_queue_type="rq"
         )
         ```
     """
@@ -249,7 +243,6 @@ def init_config(
     project_cfg = init_project_config(
         base_dir=base_dir,
         name=name,
-        job_queue_type=job_queue_type,
         fs=fs,
         storage_options=storage_options,
     )

flowerpower/cfg/base.py

@@ -2,7 +2,8 @@ import copy
 from typing import Any, Self
 
 import msgspec
-from fsspec import AbstractFileSystem, filesystem
+from fsspec_utils import AbstractFileSystem, filesystem
+from ..utils.misc import get_filesystem
 
 
 class BaseConfig(msgspec.Struct, kw_only=True):
@@ -20,12 +21,10 @@ class BaseConfig(msgspec.Struct, kw_only=True):
         Raises:
             NotImplementedError: If the filesystem does not support writing files.
         """
-        if fs is None:
-            fs = filesystem("file")
+        fs = get_filesystem(fs)
         try:
             with fs.open(path, "wb") as f:
                 f.write(msgspec.yaml.encode(self, order="deterministic"))
-                # yaml.dump(self.to_dict(), f, default_flow_style=False)
         except NotImplementedError:
             raise NotImplementedError("The filesystem does not support writing files.")
 
@@ -54,29 +53,42 @@ class BaseConfig(msgspec.Struct, kw_only=True):
             An instance of the class with the values from the YAML file.
 
         """
-        if fs is None:
-            fs = filesystem("file")
+        fs = get_filesystem(fs)
         with fs.open(path) as f:
-            # data = yaml.full_load(f)
-            # return cls.from_dict(data)
             return msgspec.yaml.decode(f.read(), type=cls, strict=False)
 
-    def update(self, d: dict[str, Any]) -> None:
+    def _apply_dict_updates(self, target: Self, d: dict[str, Any]) -> None:
+        """
+        Helper method to apply dictionary updates to a target instance.
+        
+        Args:
+            target: The target instance to apply updates to.
+            d: The dictionary containing updates to apply.
+        """
         for k, v in d.items():
-            if hasattr(self, k):
-                current_value = getattr(self, k)
+            if hasattr(target, k):
+                current_value = getattr(target, k)
                 if isinstance(current_value, dict) and isinstance(v, dict):
                     current_value.update(v)
                 else:
-                    setattr(self, k, v)
+                    setattr(target, k, v)
             else:
-                setattr(self, k, v)
+                # Use object.__setattr__ to bypass msgspec.Struct's restrictions
+                object.__setattr__(target, k, v)
+
+    def update(self, d: dict[str, Any]) -> None:
+        """
+        Updates this instance with values from the provided dictionary.
+        
+        Args:
+            d: The dictionary containing updates to apply.
+        """
+        self._apply_dict_updates(self, d)
 
     def merge_dict(self, d: dict[str, Any]) -> Self:
         """
         Creates a copy of this instance and updates the copy with values
-        from the provided dictionary, only if the dictionary field's value is not
-        its default value. The original instance (self) is not modified.
+        from the provided dictionary. The original instance (self) is not modified.
 
         Args:
             d: The dictionary to get values from.
@@ -84,16 +96,8 @@ class BaseConfig(msgspec.Struct, kw_only=True):
         Returns:
             A new instance of the struct with updated values.
         """
-        self_copy = copy.copy(self)
-        for k, v in d.items():
-            if hasattr(self_copy, k):
-                current_value = getattr(self_copy, k)
-                if isinstance(current_value, dict) and isinstance(v, dict):
-                    current_value.update(v)
-                else:
-                    setattr(self_copy, k, v)
-            else:
-                setattr(self_copy, k, v)
+        self_copy = copy.deepcopy(self)
+        self._apply_dict_updates(self_copy, d)
         return self_copy
 
     def merge(self, source: Self) -> Self:

flowerpower/cfg/pipeline/__init__.py

@@ -1,13 +1,15 @@
 import msgspec
 import yaml
+from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
 from hamilton.function_modifiers import source, value
 from munch import Munch, munchify
 
-from ...fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
 from ..base import BaseConfig
 from .adapter import AdapterConfig
+from .run import ExecutorConfig as ExecutorConfig
 from .run import RunConfig
-from .schedule import ScheduleConfig
+from .run import WithAdapterConfig as WithAdapterConfig
+#from .schedule import ScheduleConfig
 
 
 class PipelineConfig(BaseConfig):
@@ -20,7 +22,7 @@ class PipelineConfig(BaseConfig):
     Attributes:
         name (str | None): The name of the pipeline.
         run (RunConfig): Configuration for pipeline execution.
-        schedule (ScheduleConfig): Configuration for pipeline scheduling.
+        schedule (ScheduleConfig): Configuration for pipeline scheduling. DEPRECATED.
         params (dict): Pipeline parameters.
         adapter (AdapterConfig): Configuration for the pipeline adapter.
         h_params (dict): Hamilton-formatted parameters.
@@ -43,7 +45,7 @@ class PipelineConfig(BaseConfig):
 
     name: str | None = msgspec.field(default=None)
     run: RunConfig = msgspec.field(default_factory=RunConfig)
-    schedule: ScheduleConfig = msgspec.field(default_factory=ScheduleConfig)
+    #: ScheduleConfig = msgspec.field(default_factory=ScheduleConfig)
     params: dict = msgspec.field(default_factory=dict)
     adapter: AdapterConfig = msgspec.field(default_factory=AdapterConfig)
     h_params: dict = msgspec.field(default_factory=dict)
@@ -166,7 +168,7 @@ class PipelineConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(
+            fs = filesystem(
                 base_dir, cached=False, dirfs=True, storage_options=storage_options
             )
         if fs.exists("conf/pipelines"):
@@ -207,7 +209,7 @@ class PipelineConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(
+            fs = filesystem(
                 base_dir, cached=True, dirfs=True, storage_options=storage_options
             )
 

flowerpower/cfg/pipeline/_schedule.py

@@ -0,0 +1,32 @@
+import datetime as dt
+
+import msgspec
+from munch import munchify
+
+from ..base import BaseConfig
+
+
+class ScheduleConfig(BaseConfig):
+    cron: str | dict | None = msgspec.field(default=None)
+    interval: str | int | dict | None = msgspec.field(default=None)
+    date: str | None = msgspec.field(default=None)
+
+    def __post_init__(self):
+        if isinstance(self.date, str):
+            try:
+                self.date = dt.datetime.fromisoformat(self.date)
+            except ValueError:
+                raise ValueError(
+                    f"Invalid date format: {self.date}. Expected ISO format."
+                )
+        if isinstance(self.cron, dict):
+            self.cron = munchify(self.cron)
+        if isinstance(self.interval, dict):
+            self.interval = munchify(self.interval)
+
+
+# class ScheduleConfig(BaseConfig):
+#     run: ScheduleRunConfig = msgspec.field(default_factory=ScheduleRunConfig)
+#     trigger: ScheduleTriggerConfig = msgspec.field(
+#         default_factory=ScheduleTriggerConfig
+#     )

flowerpower/cfg/pipeline/adapter.py

@@ -39,17 +39,12 @@ class MLFlowConfig(BaseConfig):
             self.run_tags = munchify(self.run_tags)
 
 
-# class OpenLineageConfig(BaseConfig):
-#     namespace : str | None = msgspec.field(default=None)
-#     job_name : str | None = msgspec.field(default=None)
-
 
 class AdapterConfig(BaseConfig):
     hamilton_tracker: HamiltonTracerConfig = msgspec.field(
         default_factory=HamiltonTracerConfig
     )
     mlflow: MLFlowConfig = msgspec.field(default_factory=MLFlowConfig)
-    # openlineage: OpenLineageConfig | dict = msgspec.field(default_factory=OpenLineageConfig)
 
     def __post_init__(self):
         if isinstance(self.hamilton_tracker, dict):