FlowerPower 0.9.13.1__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

flowerpower/__init__.py

@@ -1,5 +1,20 @@
-from .flowerpower import init as init_flowerpower
-
 import importlib.metadata
 
+from .flowerpower import init as init_project  # noqa: E402
+
+from .pipeline import PipelineManager
+from .job_queue import JobQueueManager
+from .cfg import Config, ProjectConfig, PipelineConfig
+
 __version__ = importlib.metadata.version("FlowerPower")
+
+__all__ = [
+    "__version__",
+    "init_project",
+    "PipelineManager",
+    "JobQueueManager",
+    "Config",
+    "ProjectConfig",
+    "PipelineConfig",
+    "PipelineConfig",
+]

flowerpower/cfg/__init__.py

@@ -1,164 +1,96 @@
 from pathlib import Path
 
-import yaml
-from fsspec import AbstractFileSystem
-from hamilton.function_modifiers import source, value
-from munch import Munch, munchify
-from pydantic import Field
+import msgspec
+from munch import Munch
 
-from ..fs import get_filesystem
+from ..fs import AbstractFileSystem, get_filesystem
 from .base import BaseConfig
-from .pipeline.run import PipelineRunConfig
-from .pipeline.schedule import PipelineScheduleConfig
-from .pipeline.tracker import PipelineTrackerConfig
-from .project.open_telemetry import ProjectOpenTelemetryConfig
-from .project.tracker import ProjectTrackerConfig
-from .project.worker import ProjectWorkerConfig
-
-
-class PipelineConfig(BaseConfig):
-    name: str | None = None
-    run: PipelineRunConfig = Field(default_factory=PipelineRunConfig)
-    schedule: PipelineScheduleConfig = Field(default_factory=PipelineScheduleConfig)
-    params: dict | Munch = {}
-    tracker: PipelineTrackerConfig = Field(default_factory=PipelineTrackerConfig)
-    h_params: dict | Munch = {}
-
-    def model_post_init(self, __context):
-        if isinstance(self.params, dict):
-            self.h_params = munchify(self.to_h_params(self.params))
-            self.params = munchify(self.params)
-
-    def to_yaml(self, path: str, fs: AbstractFileSystem):
-        try:
-            fs.makedirs(fs._parent(path), exist_ok=True)
-            with fs.open(path, "w") as f:
-                d = self.to_dict()
-                d.pop("name")
-                d.pop("h_params")
-                yaml.dump(d, f, default_flow_style=False)
-        except NotImplementedError:
-            raise NotImplementedError(
-                "The filesystem "
-                f"{self.fs.fs.protocol[0] if isinstance(self.fs.fs.protocol, tuple) else self.fs.fs.protocol} "
-                "does not support writing files."
-            )
+from .pipeline import PipelineConfig, init_pipeline_config
+from .project import ProjectConfig, init_project_config
 
-    @classmethod
-    def from_dict(cls, name: str, d: dict | Munch):
-        d.update({"name": name})
-        return cls(**d)
 
-    @classmethod
-    def from_yaml(cls, name: str, path: str, fs: AbstractFileSystem):
-        with fs.open(path) as f:
-            return cls.from_dict(name, yaml.full_load(f))
-
-    def update(self, d: dict | Munch):
-        for k, v in d.items():
-            eval(f"self.{k}.update({v})")
-            if k == "params":
-                self.params.update(munchify(v))
-                self.h_params = munchify(self.to_h_params(self.params))
-                # self.params = munchify(self.params)
-        if "params" in d:
-            self.h_params = munchify(self.to_h_params(self.params))
-            self.params = munchify(self.params)
-
-    @staticmethod
-    # def to_h_params(d: dict) -> dict:
-    #     """Coverts a dictionary of function arguments to Hamilton function parameters"""
-    #
-    #     def transform_recursive(val, original_dict):
-    #         # If it's a dictionary, recursively transform its values
-    #         if isinstance(val, dict):
-    #             return {
-    #                 k: transform_recursive(v, original_dict) for k, v in val.items()
-    #             }
-    #         # If it's a string and matches a key in the original dictionary
-    #         elif isinstance(val, str) and val in original_dict:
-    #             return source(val)
-    #         # For all other values
-    #         else:
-    #             return value(val)
-    #
-    #     # Step 1: Replace each value with a dictionary containing key and value
-    #     result = {k: {k: d[k]} for k in d}
-    #
-    #     # Step 2 & 3: Transform all values recursively
-    #     return {k: transform_recursive(v, d) for k, v in result.items()}
-    def to_h_params(d: dict) -> dict:
-        """Converts a dictionary of function arguments to Hamilton function parameters"""
-
-        def transform_recursive(val, original_dict, depth=1):
-            if isinstance(val, dict):
-                # If we're at depth 3, wrap the entire dictionary in value()
-                if depth == 3:
-                    return value(val)
-                # Otherwise, continue recursing
-                return {
-                    k: transform_recursive(v, original_dict, depth + 1)
-                    for k, v in val.items()
-                }
-            # If it's a string and matches a key in the original dictionary
-            elif isinstance(val, str) and val in original_dict:
-                return source(val)
-            # For non-dictionary values at depth 3
-            elif depth == 3:
-                return value(val)
-            # For all other values
-            return val
-
-        # Step 1: Replace each value with a dictionary containing key and value
-        result = {k: {k: d[k]} for k in d}
-
-        # Step 2: Transform all values recursively
-        return {k: transform_recursive(v, d) for k, v in result.items()}
-
-
-class ProjectConfig(BaseConfig):
-    name: str | None = None
-    worker: ProjectWorkerConfig = Field(default_factory=ProjectWorkerConfig)
-    tracker: ProjectTrackerConfig = Field(default_factory=ProjectTrackerConfig)
-    open_telemetry: ProjectOpenTelemetryConfig = Field(
-        default_factory=ProjectOpenTelemetryConfig
-    )
+class Config(BaseConfig):
+    """Main configuration class for FlowerPower, combining project and pipeline settings.
+
+    This class serves as the central configuration manager, handling both project-wide
+    and pipeline-specific settings. It provides functionality for loading and saving
+    configurations using various filesystem abstractions.
+
+    Attributes:
+        pipeline (PipelineConfig): Configuration for the pipeline.
+        project (ProjectConfig): Configuration for the project.
+        fs (AbstractFileSystem | None): Filesystem abstraction for I/O operations.
+        base_dir (str | Path | None): Base directory for the configuration.
+        storage_options (dict | Munch): Options for filesystem operations.
+
+    Example:
+        ```python
+        # Load configuration
+        config = Config.load(
+            base_dir="my_project",
+            name="project1",
+            pipeline_name="data-pipeline"
+        )
 
+        # Save configuration
+        config.save(project=True, pipeline=True)
+        ```
+    """
 
-class Config(BaseConfig):
-    pipeline: PipelineConfig = Field(default_factory=PipelineConfig)
-    project: ProjectConfig = Field(default_factory=ProjectConfig)
+    pipeline: PipelineConfig = msgspec.field(default_factory=PipelineConfig)
+    project: ProjectConfig = msgspec.field(default_factory=ProjectConfig)
     fs: AbstractFileSystem | None = None
     base_dir: str | Path | None = None
-    storage_options: dict | Munch = Field(default_factory=Munch)
+    storage_options: dict | Munch = msgspec.field(default_factory=Munch)
 
     @classmethod
     def load(
         cls,
-        base_dir: str = "",
+        base_dir: str = ".",
         name: str | None = None,
         pipeline_name: str | None = None,
+        job_queue_type: str | None = None,
         fs: AbstractFileSystem | None = None,
         storage_options: dict | Munch = Munch(),
     ):
+        """Load both project and pipeline configurations.
+
+        Args:
+            base_dir (str, optional): Base directory for configurations. Defaults to ".".
+            name (str | None, optional): Project name. Defaults to None.
+            pipeline_name (str | None, optional): Pipeline name. Defaults to None.
+            job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
+            fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+            storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+
+        Returns:
+            Config: Combined configuration instance.
+
+        Example:
+            ```python
+            config = Config.load(
+                base_dir="my_project",
+                name="test_project",
+                pipeline_name="etl",
+                job_queue_type="rq"
+            )
+            ```
+        """
         if fs is None:
             fs = get_filesystem(base_dir, cached=True, dirfs=True, **storage_options)
-        if fs.exists("conf/project.yml"):
-            project = ProjectConfig.from_yaml(path="conf/project.yml", fs=fs)
-        else:
-            project = ProjectConfig(name=name)
-
-        if pipeline_name is not None:
-            if fs.exists(f"conf/pipelines/{pipeline_name}.yml"):
-                pipeline = PipelineConfig.from_yaml(
-                    name=pipeline_name,
-                    path=f"conf/pipelines/{pipeline_name}.yml",
-                    fs=fs,
-                )
-            else:
-                pipeline = PipelineConfig(name=pipeline_name)
-        else:
-            pipeline = PipelineConfig(name=pipeline_name)
+        project = ProjectConfig.load(
+            base_dir=base_dir,
+            name=name,
+            job_queue_type=job_queue_type,
+            fs=fs,
+            storage_options=storage_options,
+        )
+        pipeline = PipelineConfig.load(
+            base_dir=base_dir,
+            name=pipeline_name,
+            fs=fs,
+            storage_options=storage_options,
+        )
 
         return cls(
             base_dir=base_dir,
@@ -168,18 +100,44 @@ class Config(BaseConfig):
             storage_options=storage_options,
         )
 
-    def save(self):
+    def save(
+        self,
+        project: bool = False,
+        pipeline: bool = True,
+        fs: AbstractFileSystem | None = None,
+        storage_options: dict | Munch = Munch(),
+    ):
+        """Save project and/or pipeline configurations.
+
+        Args:
+            project (bool, optional): Whether to save project config. Defaults to False.
+            pipeline (bool, optional): Whether to save pipeline config. Defaults to True.
+            fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+            storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+
+        Example:
+            ```python
+            config.save(project=True, pipeline=True)
+            ```
+        """
+        if fs is None and self.fs is None:
+            self.fs = get_filesystem(
+                self.base_dir, cached=True, dirfs=True, **storage_options
+            )
+
         if not self.fs.exists("conf"):
             self.fs.makedirs("conf")
 
-        if self.pipeline.name is not None:
-            # h_params = self.pipeline.params.pop("h_params") if "h_params" in self.pipeline.params else None
+        if pipeline:
+            self.fs.makedirs("conf/pipelines", exist_ok=True)
+            h_params = self.pipeline.pop("h_params") if self.pipeline.h_params else None
             self.pipeline.to_yaml(
-                f"conf/pipelines/{self.pipeline.name.replace('.', '/')}.yml", self.fs
+                path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
             )
-            # if h_params is not None:
-            #    self.pipeline.params["h_params"] = h_params
-        self.project.to_yaml("conf/project.yml", self.fs)
+            if h_params:
+                self.pipeline.h_params = h_params
+        if project:
+            self.project.to_yaml("conf/project.yml", self.fs)
 
 
 def load(
@@ -189,6 +147,25 @@ def load(
     storage_options: dict | Munch = Munch(),
     fs: AbstractFileSystem | None = None,
 ):
+    """Helper function to load configuration.
+
+    This is a convenience wrapper around Config.load().
+
+    Args:
+        base_dir (str): Base directory for configurations.
+        name (str | None, optional): Project name. Defaults to None.
+        pipeline_name (str | None, optional): Pipeline name. Defaults to None.
+        storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+        fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+
+    Returns:
+        Config: Combined configuration instance.
+
+    Example:
+        ```python
+        config = load(base_dir="my_project", name="test", pipeline_name="etl")
+        ```
+    """
     return Config.load(
         name=name,
         pipeline_name=pipeline_name,
@@ -198,5 +175,80 @@ def load(
     )
 
 
-def save(config: Config):
-    config.save()
+def save(
+    config: Config,
+    project: bool = False,
+    pipeline: bool = True,
+    fs: AbstractFileSystem | None = None,
+    storage_options: dict | Munch = Munch(),
+):
+    """Helper function to save configuration.
+
+    This is a convenience wrapper around Config.save().
+
+    Args:
+        config (Config): Configuration instance to save.
+        project (bool, optional): Whether to save project config. Defaults to False.
+        pipeline (bool, optional): Whether to save pipeline config. Defaults to True.
+        fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+        storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+
+    Example:
+        ```python
+        config = load(base_dir="my_project")
+        save(config, project=True, pipeline=True)
+        ```
+    """
+    config.save(
+        project=project, pipeline=pipeline, fs=fs, storage_options=storage_options
+    )
+
+
+def init_config(
+    base_dir: str = ".",
+    name: str | None = None,
+    pipeline_name: str | None = None,
+    job_queue_type: str | None = None,
+    fs: AbstractFileSystem | None = None,
+    storage_options: dict | Munch = Munch(),
+):
+    """Initialize a new configuration with both project and pipeline settings.
+
+    This function creates and initializes both project and pipeline configurations,
+    combining them into a single Config instance.
+
+    Args:
+        base_dir (str, optional): Base directory for configurations. Defaults to ".".
+        name (str | None, optional): Project name. Defaults to None.
+        pipeline_name (str | None, optional): Pipeline name. Defaults to None.
+        job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
+        fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+        storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+
+    Returns:
+        Config: The initialized configuration instance.
+
+    Example:
+        ```python
+        config = init_config(
+            base_dir="my_project",
+            name="test_project",
+            pipeline_name="data-pipeline",
+            job_queue_type="rq"
+        )
+        ```
+    """
+    pipeline_cfg = init_pipeline_config(
+        base_dir=base_dir,
+        name=pipeline_name,
+        fs=fs,
+        storage_options=storage_options,
+    )
+    project_cfg = init_project_config(
+        base_dir=base_dir,
+        name=name,
+        job_queue_type=job_queue_type,
+        fs=fs,
+        storage_options=storage_options,
+    )
+    return Config(pipeline=pipeline_cfg, project=project_cfg, fs=fs, base_dir=base_dir)

flowerpower/cfg/base.py

@@ -1,39 +1,137 @@
-from typing import Any
+import copy
+from typing import Any, Self
 
-import yaml
-from fsspec import AbstractFileSystem
-from munch import Munch, unmunchify
-from pydantic import BaseModel, ConfigDict
+import msgspec
+from fsspec import AbstractFileSystem, filesystem
 
 
-class BaseConfig(BaseModel):
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
+class BaseConfig(msgspec.Struct, kw_only=True):
     def to_dict(self) -> dict[str, Any]:
-        return unmunchify(self.model_dump())
+        return msgspec.to_builtins(self)
 
     def to_yaml(self, path: str, fs: AbstractFileSystem | None = None) -> None:
+        """
+        Converts the instance to a YAML file.
+
+        Args:
+            path: The path to the YAML file.
+            fs: An optional filesystem instance to use for file operations.
+
+        Raises:
+            NotImplementedError: If the filesystem does not support writing files.
+        """
+        if fs is None:
+            fs = filesystem("file")
         try:
-            with fs.open(path, "w") as f:
-                yaml.dump(self.to_dict(), f, default_flow_style=False)
+            with fs.open(path, "wb") as f:
+                f.write(msgspec.yaml.encode(self, order="deterministic"))
+                # yaml.dump(self.to_dict(), f, default_flow_style=False)
         except NotImplementedError:
-            raise NotImplementedError(
-                "The filesystem "
-                f"{self.fs.fs.protocol[0] if isinstance(self.fs.fs.protocol, tuple) else self.fs.fs.protocol} "
-                "does not support writing files."
-            )
+            raise NotImplementedError("The filesystem does not support writing files.")
 
     @classmethod
-    def from_dict(cls, d: dict[str, Any] | Munch) -> "BaseConfig":
-        return cls(**d)
+    def from_dict(cls, data: dict[str, Any]) -> "BaseConfig":
+        """
+        Converts a dictionary to an instance of the class.
+        Args:
+            data: The dictionary to convert.
+
+        Returns:
+            An instance of the class with the values from the dictionary.
+        """
+        return msgspec.convert(data, cls)
 
     @classmethod
-    def from_yaml(cls, path: str, fs: AbstractFileSystem):
-        # if fs is None:
-        #    fs = get_filesystem(".", cached=True)
+    def from_yaml(cls, path: str, fs: AbstractFileSystem | None = None) -> "BaseConfig":
+        """
+        Loads a YAML file and converts it to an instance of the class.
+
+        Args:
+            path: The path to the YAML file.
+            fs: An optional filesystem instance to use for file operations.
+
+        Returns:
+            An instance of the class with the values from the YAML file.
+
+        """
+        if fs is None:
+            fs = filesystem("file")
         with fs.open(path) as f:
-            return cls.from_dict(yaml.full_load(f))
+            # data = yaml.full_load(f)
+            # return cls.from_dict(data)
+            return msgspec.yaml.decode(f.read(), type=cls, strict=False)
+
+    def update(self, d: dict[str, Any]) -> None:
+        for k, v in d.items():
+            if hasattr(self, k):
+                current_value = getattr(self, k)
+                if isinstance(current_value, dict) and isinstance(v, dict):
+                    current_value.update(v)
+                else:
+                    setattr(self, k, v)
+            else:
+                setattr(self, k, v)
+
+    def merge_dict(self, d: dict[str, Any]) -> Self:
+        """
+        Creates a copy of this instance and updates the copy with values
+        from the provided dictionary, only if the dictionary field's value is not
+        its default value. The original instance (self) is not modified.
+
+        Args:
+            d: The dictionary to get values from.
 
-    def update(self, d: dict[str, Any] | Munch) -> None:
+        Returns:
+            A new instance of the struct with updated values.
+        """
+        self_copy = copy.copy(self)
         for k, v in d.items():
-            setattr(self, k, v)
+            if hasattr(self_copy, k):
+                current_value = getattr(self_copy, k)
+                if isinstance(current_value, dict) and isinstance(v, dict):
+                    current_value.update(v)
+                else:
+                    setattr(self_copy, k, v)
+            else:
+                setattr(self_copy, k, v)
+        return self_copy
+
+    def merge(self, source: Self) -> Self:
+        """
+        Creates a copy of this instance and updates the copy with values
+        from the source struct, only if the source field's value is not
+        its default value. The original instance (self) is not modified.
+
+        Args:
+            source: The msgspec.Struct instance of the same type to get values from.
+
+        Returns:
+            A new instance of the struct with updated values.
+
+        Raises:
+            TypeError: If source is not of the same type as self.
+        """
+        if type(self) is not type(source):
+            raise TypeError(
+                f"Source must be an instance of {type(self).__name__}, not {type(source).__name__}"
+            )
+
+        updated_instance = copy.copy(self)
+
+        # Get default values if they exist
+        defaults = getattr(source, "__struct_defaults__", {})
+
+        for field in source.__struct_fields__:
+            source_value = getattr(source, field)
+            has_explicit_default = field in defaults
+            is_default_value = False
+
+            if has_explicit_default:
+                is_default_value = source_value == defaults[field]
+            else:
+                is_default_value = source_value is None
+
+            if not is_default_value:
+                setattr(updated_instance, field, source_value)
+
+        return updated_instance