FlowerPower 0.9.13.1__py3-none-any.whl → 1.0.0b2__py3-none-any.whl

flowerpower/cfg/pipeline/__init__.py

@@ -0,0 +1,254 @@
+import msgspec
+import yaml
+from hamilton.function_modifiers import source, value
+from munch import Munch, munchify
+
+from ...fs import AbstractFileSystem, get_filesystem
+from ..base import BaseConfig
+from .adapter import AdapterConfig
+from .run import RunConfig
+from .schedule import ScheduleConfig
+
+
+class PipelineConfig(BaseConfig):
+    """Configuration class for managing pipeline settings in FlowerPower.
+
+    This class handles pipeline-specific configuration including run settings, scheduling,
+    parameters, and adapter settings. It supports Hamilton-style parameter configuration
+    and YAML serialization.
+
+    Attributes:
+        name (str | None): The name of the pipeline.
+        run (RunConfig): Configuration for pipeline execution.
+        schedule (ScheduleConfig): Configuration for pipeline scheduling.
+        params (dict): Pipeline parameters.
+        adapter (AdapterConfig): Configuration for the pipeline adapter.
+        h_params (dict): Hamilton-formatted parameters.
+
+    Example:
+        ```python
+        # Create a new pipeline config
+        pipeline = PipelineConfig(name="data-transform")
+
+        # Set parameters
+        pipeline.params = {
+            "input_path": "data/input",
+            "batch_size": 100
+        }
+
+        # Save configuration
+        pipeline.save(name="data-transform")
+        ```
+    """
+
+    name: str | None = msgspec.field(default=None)
+    run: RunConfig = msgspec.field(default_factory=RunConfig)
+    schedule: ScheduleConfig = msgspec.field(default_factory=ScheduleConfig)
+    params: dict = msgspec.field(default_factory=dict)
+    adapter: AdapterConfig = msgspec.field(default_factory=AdapterConfig)
+    h_params: dict = msgspec.field(default_factory=dict)
+
+    def __post_init__(self):
+        if isinstance(self.params, dict):
+            self.h_params = munchify(self.to_h_params(self.params))
+            self.params = munchify(self.params)
+
+    def to_yaml(self, path: str, fs: AbstractFileSystem):
+        try:
+            fs.makedirs(fs._parent(path), exist_ok=True)
+            with fs.open(path, "w") as f:
+                d = self.to_dict()
+                d.pop("name")
+                d.pop("h_params")
+                yaml.dump(d, f, default_flow_style=False)
+        except NotImplementedError:
+            raise NotImplementedError(
+                "The filesystem "
+                f"{self.fs.fs.protocol[0] if isinstance(self.fs.fs.protocol, tuple) else self.fs.fs.protocol} "
+                "does not support writing files."
+            )
+
+    @classmethod
+    def from_dict(cls, name: str, data: dict | Munch):
+        data.update({"name": name})
+        return msgspec.convert(data, cls)
+
+    @classmethod
+    def from_yaml(cls, name: str, path: str, fs: AbstractFileSystem):
+        with fs.open(path) as f:
+            data = yaml.full_load(f)
+            return cls.from_dict(name=name, data=data)
+
+    def update(self, d: dict | Munch):
+        for k, v in d.items():
+            eval(f"self.{k}.update({v})")
+            if k == "params":
+                self.params.update(munchify(v))
+                self.h_params = munchify(self.to_h_params(self.params))
+                # self.params = munchify(self.params)
+        if "params" in d:
+            self.h_params = munchify(self.to_h_params(self.params))
+            self.params = munchify(self.params)
+
+    @staticmethod
+    def to_h_params(d: dict) -> dict:
+        """Convert a dictionary of parameters to Hamilton-compatible format.
+
+        This method transforms regular parameter dictionaries into Hamilton's function parameter
+        format, supporting nested parameters and source/value decorators.
+
+        Args:
+            d (dict): The input parameter dictionary.
+
+        Returns:
+            dict: Hamilton-formatted parameter dictionary.
+
+        Example:
+            ```python
+            params = {
+                "batch_size": 100,
+                "paths": {"input": "data/in", "output": "data/out"}
+            }
+            h_params = PipelineConfig.to_h_params(params)
+            ```
+        """
+
+        def transform_recursive(val, original_dict, depth=1):
+            if isinstance(val, dict):
+                # If we're at depth 3, wrap the entire dictionary in value()
+                if depth == 3:
+                    return value(val)
+                # Otherwise, continue recursing
+                return {
+                    k: transform_recursive(v, original_dict, depth + 1)
+                    for k, v in val.items()
+                }
+            # If it's a string and matches a key in the original dictionary
+            elif isinstance(val, str) and val in original_dict:
+                return source(val)
+            # For non-dictionary values at depth 3
+            elif depth == 3:
+                return value(val)
+            # For all other values
+            return val
+
+        # Step 1: Replace each value with a dictionary containing key and value
+        result = {k: {k: d[k]} for k in d}
+
+        # Step 2: Transform all values recursively
+        return {k: transform_recursive(v, d) for k, v in result.items()}
+
+    @classmethod
+    def load(
+        cls,
+        base_dir: str = ".",
+        name: str | None = None,
+        fs: AbstractFileSystem | None = None,
+        storage_options: dict | Munch = Munch(),
+    ):
+        """Load pipeline configuration from a YAML file.
+
+        Args:
+            base_dir (str, optional): Base directory for the pipeline. Defaults to ".".
+            name (str | None, optional): Pipeline name. Defaults to None.
+            fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+            storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+
+        Returns:
+            PipelineConfig: Loaded pipeline configuration.
+
+        Example:
+            ```python
+            pipeline = PipelineConfig.load(
+                base_dir="my_project",
+                name="data-pipeline"
+            )
+            ```
+        """
+        if fs is None:
+            fs = get_filesystem(base_dir, cached=False, dirfs=True, **storage_options)
+        if fs.exists("conf/pipelines"):
+            if name is not None:
+                pipeline = PipelineConfig.from_yaml(
+                    name=name,
+                    path=f"conf/pipelines/{name}.yml",
+                    fs=fs,
+                )
+            else:
+                pipeline = PipelineConfig(name=name)
+        else:
+            pipeline = PipelineConfig(name=name)
+
+        return pipeline
+
+    def save(
+        self,
+        name: str | None = None,
+        base_dir: str = ".",
+        fs: AbstractFileSystem | None = None,
+        storage_options: dict | Munch = Munch(),
+    ):
+        """Save pipeline configuration to a YAML file.
+
+        Args:
+            name (str | None, optional): Pipeline name. Defaults to None.
+            base_dir (str, optional): Base directory for the pipeline. Defaults to ".".
+            fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+            storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+
+        Raises:
+            ValueError: If pipeline name is not set.
+
+        Example:
+            ```python
+            pipeline_config.save(name="data-pipeline", base_dir="my_project")
+            ```
+        """
+        if fs is None:
+            fs = get_filesystem(base_dir, cached=True, dirfs=True, **storage_options)
+
+        fs.makedirs("conf/pipelines", exist_ok=True)
+        if name is not None:
+            self.name = name
+        if self.name is None:
+            raise ValueError("Pipeline name is not set. Please provide a name.")
+
+        h_params = getattr(self, "h_params")
+
+        self.to_yaml(path=f"conf/pipelines/{self.name}.yml", fs=fs)
+
+        setattr(self, "h_params", h_params)
+
+
+def init_pipeline_config(
+    base_dir: str = ".",
+    name: str | None = None,
+    fs: AbstractFileSystem | None = None,
+    storage_options: dict | Munch = Munch(),
+):
+    """Initialize a new pipeline configuration.
+
+    This function creates a new pipeline configuration and saves it to disk.
+
+    Args:
+        base_dir (str, optional): Base directory for the pipeline. Defaults to ".".
+        name (str | None, optional): Pipeline name. Defaults to None.
+        fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+        storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+
+    Returns:
+        PipelineConfig: The initialized pipeline configuration.
+
+    Example:
+        ```python
+        pipeline = init_pipeline_config(
+            base_dir="my_project",
+            name="etl-pipeline"
+        )
+        ```
+    """
+    pipeline = PipelineConfig.load(
+        base_dir=base_dir, name=name, fs=fs, storage_options=storage_options
+    )
+    pipeline.save(name=name, base_dir=base_dir, fs=fs, storage_options=storage_options)
+    return pipeline

flowerpower/cfg/pipeline/adapter.py

@@ -0,0 +1,66 @@
+import msgspec
+from munch import munchify
+
+from ... import settings
+from ..base import BaseConfig
+
+
+class HamiltonTracerConfig(BaseConfig):
+    project_id: int | None = msgspec.field(default=None)
+    dag_name: str | None = msgspec.field(default=None)
+    tags: dict = msgspec.field(default_factory=dict)
+    capture_data_statistics: bool = msgspec.field(
+        default=settings.HAMILTON_CAPTURE_DATA_STATISTICS
+    )
+    max_list_length_capture: int = msgspec.field(
+        default=settings.HAMILTON_MAX_LIST_LENGTH_CAPTURE
+    )
+    max_dict_length_capture: int = msgspec.field(
+        default=settings.HAMILTON_MAX_DICT_LENGTH_CAPTURE
+    )
+
+    def __post_init__(self):
+        self.tags = munchify(self.tags)
+
+
+class MLFlowConfig(BaseConfig):
+    experiment_name: str | None = msgspec.field(default=None)
+    experiment_tags: dict | None = msgspec.field(default_factory=dict)
+    experiment_description: str | None = msgspec.field(default=None)
+    run_id: str | None = msgspec.field(default=None)
+    run_name: str | None = msgspec.field(default=None)
+    run_tags: dict | None = msgspec.field(default_factory=dict)
+    run_description: str | None = msgspec.field(default=None)
+
+    def __post_init__(self):
+        if isinstance(self.experiment_tags, dict):
+            self.experiment_tags = munchify(self.experiment_tags)
+        if isinstance(self.run_tags, dict):
+            self.run_tags = munchify(self.run_tags)
+
+
+# class OpenLineageConfig(BaseConfig):
+#     namespace : str | None = msgspec.field(default=None)
+#     job_name : str | None = msgspec.field(default=None)
+
+
+class AdapterConfig(BaseConfig):
+    hamilton_tracker: HamiltonTracerConfig = msgspec.field(
+        default_factory=HamiltonTracerConfig
+    )
+    mlflow: MLFlowConfig = msgspec.field(default_factory=MLFlowConfig)
+    # openlineage: OpenLineageConfig | dict = msgspec.field(default_factory=OpenLineageConfig)
+
+    def __post_init__(self):
+        if isinstance(self.hamilton_tracker, dict):
+            self.hamilton_tracker = HamiltonTracerConfig.from_dict(
+                self.hamilton_tracker
+            )
+        if isinstance(self.mlflow, dict):
+            self.mlflow = MLFlowConfig.from_dict(self.mlflow)
+        if self.hamilton_tracker.project_id is not None:
+            self.mlflow.experiment_name = (
+                f"flowerpower_project_{self.hamilton_tracker.project_id}"
+            )
+        if self.hamilton_tracker.dag_name is not None:
+            self.mlflow.run_name = self.hamilton_tracker.dag_name

flowerpower/cfg/pipeline/run.py

@@ -1,18 +1,47 @@
-from munch import Munch, munchify
-from pydantic import Field
+import msgspec
+from munch import munchify
 
+from ... import settings
 from ..base import BaseConfig
 
 
-class PipelineRunConfig(BaseConfig):
-    final_vars: list[str] = Field(default_factory=list)
-    inputs: dict | Munch = Field(default_factory=dict)
-    executor: str | None = None
-    config: dict | Munch = Field(default_factory=dict)
-    with_tracker: bool = False
-    with_opentelemetry: bool = False
-    with_progressbar: bool = False
+class WithAdapterConfig(BaseConfig):
+    tracker: bool = msgspec.field(default=False)
+    mlflow: bool = msgspec.field(default=False)
+    # openlineage: bool = msgspec.field(default=False)
+    ray: bool = msgspec.field(default=False)
+    opentelemetry: bool = msgspec.field(default=False)
+    progressbar: bool = msgspec.field(default=False)
+    future: bool = msgspec.field(default=False)
 
-    def model_post_init(self, __context):
+
+class ExecutorConfig(BaseConfig):
+    type: str | None = msgspec.field(default=settings.EXECUTOR)
+    max_workers: int | None = msgspec.field(default=settings.EXECUTOR_MAX_WORKERS)
+    num_cpus: int | None = msgspec.field(default=settings.EXECUTOR_NUM_CPUS)
+
+
+class RunConfig(BaseConfig):
+    inputs: dict | None = msgspec.field(default_factory=dict)
+    final_vars: list[str] | None = msgspec.field(default_factory=list)
+    config: dict | None = msgspec.field(default_factory=dict)
+    cache: dict | bool | None = msgspec.field(default=False)
+    with_adapter: WithAdapterConfig = msgspec.field(default_factory=WithAdapterConfig)
+    executor: ExecutorConfig = msgspec.field(default_factory=ExecutorConfig)
+    log_level: str | None = msgspec.field(default=None)
+    max_retries: int = msgspec.field(default=3)
+    retry_delay: int | float = msgspec.field(default=1)
+    jitter_factor: float | None = msgspec.field(default=0.1)
+    retry_exceptions: tuple[str] = msgspec.field(default_factory=lambda: ("Exception",))
+
+    def __post_init__(self):
         if isinstance(self.inputs, dict):
             self.inputs = munchify(self.inputs)
+        if isinstance(self.config, dict):
+            self.config = munchify(self.config)
+        if isinstance(self.cache, (dict)):
+            self.cache = munchify(self.cache)
+        if isinstance(self.with_adapter, dict):
+            self.with_adapter = WithAdapterConfig.from_dict(self.with_adapter)
+        if isinstance(self.executor, dict):
+            self.executor = ExecutorConfig.from_dict(self.executor)

flowerpower/cfg/pipeline/schedule.py

@@ -1,84 +1,74 @@
 import datetime as dt
 
-from pydantic import Field
+import msgspec
+from munch import munchify
 
 from ..base import BaseConfig
 
-
-class PipelineScheduleCronTriggerConfig(BaseConfig):
-    year: str | int | None = None
-    month: str | int | None = None
-    week: str | int | None = None
-    day: str | int | None = None
-    day_of_week: str | int | None = None
-    hour: str | int | None = None
-    minute: str | int | None = None
-    second: str | int | None = None
-    start_time: dt.datetime | None = None
-    end_time: dt.datetime | None = None
-    timezone: str | None = None
-    crontab: str | None = None
-
-
-class PipelineScheduleIntervalTriggerConfig(BaseConfig):
-    weeks: int | float | None = None
-    days: int | float | None = None
-    hours: int | float | None = None
-    minutes: int | float | None = None
-    seconds: int | float | None = None
-    microseconds: int | float | None = None
-    start_time: dt.datetime | None = None
-    end_time: dt.datetime | None = None
-
-
-class PipelineScheduleCalendarTriggerConfig(BaseConfig):
-    years: int | float | None = None
-    months: int | float | None = None
-    weeks: int | float | None = None
-    days: int | float | None = None
-    hour: int | float | None = None
-    minute: int | float | None = None
-    second: int | float | None = None
-    start_date: dt.datetime | None = None
-    end_date: dt.datetime | None = None
-    timezone: str | None = None
-
-
-class PipelineScheduleDateTriggerConfig(BaseConfig):
-    run_time: dt.datetime | None = None
-
-
-class PipelineScheduleTriggerConfig(BaseConfig):
-    cron: PipelineScheduleCronTriggerConfig = Field(
-        default_factory=PipelineScheduleCronTriggerConfig
-    )
-    interval: PipelineScheduleIntervalTriggerConfig = Field(
-        default_factory=PipelineScheduleIntervalTriggerConfig
-    )
-    calendar: PipelineScheduleCalendarTriggerConfig = Field(
-        default_factory=PipelineScheduleCalendarTriggerConfig
-    )
-    date: PipelineScheduleDateTriggerConfig = Field(
-        default_factory=PipelineScheduleDateTriggerConfig
-    )
-    type_: str | None = None
-
-
-class PipelineScheduleRunConfig(BaseConfig):
-    id_: str | None = None
-    executor: str | None = None
-    paused: bool = False
-    coalesce: str = "latest"  # other options are "all" and "earliest"
-    misfire_grace_time: int | float | dt.timedelta | None = None
-    max_jitter: int | float | dt.timedelta | None = None
-    max_running_jobs: int | None = None
-    conflict_policy: str | None = (
-        "do_nothing"  # other options are "replace" and "exception"
-    )
-
-
-class PipelineScheduleConfig(BaseConfig):
-    run: PipelineScheduleRunConfig = Field(default_factory=PipelineScheduleRunConfig)
-    trigger: PipelineScheduleTriggerConfig = Field(
-        default_factory=PipelineScheduleTriggerConfig
-    )
+# class ScheduleCronTriggerConfig(BaseConfig):
+#     year: str | int | None = None
+#     month: str | int | None = None
+#     week: str | int | None = None
+#     day: str | int | None = None
+#     day_of_week: str | int | None = None
+#     hour: str | int | None = None
+#     minute: str | int | None = None
+#     second: str | int | None = None
+#     start_time: dt.datetime | None = None
+#     end_time: dt.datetime | None = None
+#     timezone: str | None = None
+#     crontab: str | None = None
+
+
+# class ScheduleIntervalTriggerConfig(BaseConfig):
+#     weeks: int | float | None = None
+#     days: int | float | None = None
+#     hours: int | float | None = None
+#     minutes: int | float | None = None
+#     seconds: int | float | None = None
+#     microseconds: int | float | None = None
+#     start_time: dt.datetime | None = None
+#     end_time: dt.datetime | None = None
+
+
+# class ScheduleCalendarTriggerConfig(BaseConfig):
+#     years: int | float | None = None
+#     months: int | float | None = None
+#     weeks: int | float | None = None
+#     days: int | float | None = None
+#     hour: int | float | None = None
+#     minute: int | float | None = None
+#     second: int | float | None = None
+#     start_date: dt.datetime | None = None
+#     end_date: dt.datetime | None = None
+#     timezone: str | None = None
+
+
+# class ScheduleDateTriggerConfig(BaseConfig):
+#     run_time: dt.datetime | None = None
+
+
+class ScheduleConfig(BaseConfig):
+    cron: str | dict | None = msgspec.field(default=None)
+    interval: str | int | dict | None = msgspec.field(default=None)
+    date: str | None = msgspec.field(default=None)
+
+    def __post_init__(self):
+        if isinstance(self.date, str):
+            try:
+                self.date = dt.datetime.fromisoformat(self.date)
+            except ValueError:
+                raise ValueError(
+                    f"Invalid date format: {self.date}. Expected ISO format."
+                )
+        if isinstance(self.cron, dict):
+            self.cron = munchify(self.cron)
+        if isinstance(self.interval, dict):
+            self.interval = munchify(self.interval)
+
+
+# class ScheduleConfig(BaseConfig):
+#     run: ScheduleRunConfig = msgspec.field(default_factory=ScheduleRunConfig)
+#     trigger: ScheduleTriggerConfig = msgspec.field(
+#         default_factory=ScheduleTriggerConfig
+#     )

flowerpower/cfg/project/__init__.py

@@ -0,0 +1,149 @@
+import msgspec
+from munch import Munch
+
+from ...fs import AbstractFileSystem, get_filesystem
+from ..base import BaseConfig
+from .adapter import AdapterConfig
+from .job_queue import JobQueueConfig
+
+
+class ProjectConfig(BaseConfig):
+    """A configuration class for managing project-level settings in FlowerPower.
+
+    This class handles project-wide configuration including job queue and adapter settings.
+    It supports loading from and saving to YAML files, with filesystem abstraction.
+
+    Attributes:
+        name (str | None): The name of the project.
+        job_queue (JobQueueConfig): Configuration for the job queue component.
+        adapter (AdapterConfig): Configuration for the adapter component.
+
+    Example:
+        ```python
+        # Create a new project config
+        project = ProjectConfig(name="my-project")
+
+        # Load existing project config
+        project = ProjectConfig.load(base_dir="path/to/project")
+
+        # Save project config
+        project.save(base_dir="path/to/project")
+        ```
+    """
+
+    name: str | None = msgspec.field(default=None)
+    job_queue: JobQueueConfig = msgspec.field(default_factory=JobQueueConfig)
+    adapter: AdapterConfig = msgspec.field(default_factory=AdapterConfig)
+
+    def __post_init__(self):
+        if isinstance(self.job_queue, dict):
+            self.job_queue = JobQueueConfig.from_dict(self.job_queue)
+        if isinstance(self.adapter, dict):
+            self.adapter = AdapterConfig.from_dict(self.adapter)
+
+    @classmethod
+    def load(
+        cls,
+        base_dir: str = ".",
+        name: str | None = None,
+        job_queue_type: str | None = None,
+        fs: AbstractFileSystem | None = None,
+        storage_options: dict | Munch = Munch(),
+    ):
+        """Load project configuration from a YAML file.
+
+        Args:
+            base_dir (str, optional): Base directory for the project. Defaults to ".".
+            name (str | None, optional): Project name. Defaults to None.
+            job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
+            fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+            storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+
+        Returns:
+            ProjectConfig: Loaded project configuration.
+
+        Example:
+            ```python
+            project = ProjectConfig.load(
+                base_dir="my_project",
+                name="pipeline1",
+                job_queue_type="rq"
+                )
+            ```
+        """
+        if fs is None:
+            fs = get_filesystem(base_dir, cached=False, dirfs=True, **storage_options)
+        if fs.exists("conf/project.yml"):
+            project = ProjectConfig.from_yaml(path="conf/project.yml", fs=fs)
+        else:
+            project = ProjectConfig(name=name)
+        if job_queue_type is not None:
+            if job_queue_type != project.job_queue.type:
+                project.job_queue.update_type(job_queue_type)
+
+        return project
+
+    def save(
+        self,
+        base_dir: str = ".",
+        fs: AbstractFileSystem | None = None,
+        storage_options: dict | Munch = Munch(),
+    ):
+        """Save project configuration to a YAML file.
+
+        Args:
+            base_dir (str, optional): Base directory for the project. Defaults to ".".
+            fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+            storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+
+        Example:
+            ```python
+            project_config.save(base_dir="my_project")
+            ```
+        """
+        if fs is None:
+            fs = get_filesystem(base_dir, cached=True, dirfs=True, **storage_options)
+
+        fs.makedirs("conf", exist_ok=True)
+        self.to_yaml(path="conf/project.yml", fs=fs)
+
+
+def init_project_config(
+    base_dir: str = ".",
+    name: str | None = None,
+    job_queue_type: str | None = None,
+    fs: AbstractFileSystem | None = None,
+    storage_options: dict | Munch = Munch(),
+):
+    """Initialize a new project configuration.
+
+    This function creates a new project configuration and saves it to disk.
+
+    Args:
+        base_dir (str, optional): Base directory for the project. Defaults to ".".
+        name (str | None, optional): Project name. Defaults to None.
+        job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
+        fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
+        storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
+
+    Returns:
+        ProjectConfig: The initialized project configuration.
+
+    Example:
+        ```python
+        project = init_project_config(
+            base_dir="my_project",
+            name="test_project",
+            job_queue_type="rq"
+        )
+        ```
+    """
+    project = ProjectConfig.load(
+        base_dir=base_dir,
+        name=name,
+        job_queue_type=job_queue_type,
+        fs=fs,
+        storage_options=storage_options,
+    )
+    project.save(base_dir=base_dir, fs=fs, storage_options=storage_options)
+    return project