cloe-nessy 0.3.18__py3-none-any.whl → 1.0.0__py3-none-any.whl

cloe_nessy/pipeline/actions/transform_with_column.py

@@ -0,0 +1,104 @@
+"""Transform action to add or update a column using a SQL expression."""
+
+from typing import Any
+
+from pyspark.sql import functions as F
+
+from cloe_nessy.pipeline.pipeline_action import PipelineAction
+from cloe_nessy.pipeline.pipeline_context import PipelineContext
+
+
+class TransformWithColumnAction(PipelineAction):
+    """Add or update a column in the DataFrame using a SQL expression.
+
+    This action uses PySpark's expr() function to evaluate SQL expressions and
+    create or update columns in the DataFrame.
+
+    Examples:
+        === "Create new column"
+            ```yaml
+            Create Full Name:
+                action: TRANSFORM_WITH_COLUMN
+                options:
+                    column_name: full_name
+                    expression: concat(first_name, ' ', last_name)
+            ```
+
+        === "Update existing column"
+            ```yaml
+            Lowercase Email:
+                action: TRANSFORM_WITH_COLUMN
+                options:
+                    column_name: email
+                    expression: lower(email)
+            ```
+
+        === "Calculated column"
+            ```yaml
+            Calculate Total:
+                action: TRANSFORM_WITH_COLUMN
+                options:
+                    column_name: total_price
+                    expression: price * quantity * (1 + tax_rate)
+            ```
+
+        === "Extract date parts"
+            ```yaml
+            Extract Year:
+                action: TRANSFORM_WITH_COLUMN
+                options:
+                    column_name: year
+                    expression: year(order_date)
+            ```
+    """
+
+    name: str = "TRANSFORM_WITH_COLUMN"
+
+    def run(
+        self,
+        context: PipelineContext,
+        *,
+        column_name: str = "",
+        expression: str = "",
+        **_: Any,
+    ) -> PipelineContext:
+        """Add or update a column using a SQL expression.
+
+        Args:
+            context: The pipeline context containing the DataFrame
+            column_name: Name of the column to create or update
+            expression: SQL expression to evaluate for the column value
+            **_: Additional unused keyword arguments
+
+        Returns:
+            PipelineContext: Updated context with the modified DataFrame
+
+        Raises:
+            ValueError: If column_name is not provided
+            ValueError: If expression is not provided
+            ValueError: If context.data is None
+            Exception: If the SQL expression is invalid
+        """
+        if not column_name:
+            raise ValueError("No column_name provided.")
+
+        if not expression:
+            raise ValueError("No expression provided.")
+
+        if context.data is None:
+            raise ValueError("Data from context is required for transform_with_column")
+
+        self._console_logger.info(f"Adding/updating column '{column_name}' with expression: {expression}")
+
+        df = context.data
+
+        try:
+            # Use F.expr() to evaluate the SQL expression
+            df = df.withColumn(column_name, F.expr(expression))
+        except Exception as e:
+            self._console_logger.error(f"Failed to evaluate expression '{expression}' for column '{column_name}': {e}")
+            raise
+
+        self._console_logger.info(f"Successfully added/updated column '{column_name}'")
+
+        return context.from_existing(data=df)

cloe_nessy/pipeline/actions/write_delta_append.py

@@ -19,9 +19,6 @@ class WriteDeltaAppendAction(PipelineAction):
                 table_identifier: my_catalog.my_schema.my_table
                 ignore_empty_df: false
         ```
-
-    Returns:
-        None.
     """
 
     name: str = "WRITE_DELTA_APPEND"

cloe_nessy/pipeline/actions/write_delta_merge.py

@@ -28,9 +28,6 @@ class WriteDeltaMergeAction(PipelineAction):
                 when_not_matched_insert: true
                 use_partition_pruning: true
         ```
-
-    Returns:
-        None.
     """
 
     name: str = "WRITE_DELTA_MERGE"

cloe_nessy/pipeline/pipeline_builder.py

@@ -0,0 +1,210 @@
+from collections import OrderedDict
+from collections.abc import Callable
+from typing import Any, Self
+
+from .pipeline import Pipeline
+from .pipeline_step import PipelineStep
+
+
+class PipelineBuilder:
+    """Fluent API builder for creating Nessy pipelines programmatically.
+
+    This class provides a chainable interface for building pipelines using method calls
+    instead of YAML configuration. It dynamically creates methods for all available
+    PipelineActions.
+
+    Example:
+        ```python
+        pipeline = (PipelineBuilder("My Pipeline")
+            .read_files(location="data/*.csv", extension="csv")
+            .transform_clean_column_names()
+            .transform_filter(condition="amount > 1000")
+            .write_catalog_table(catalog="prod", schema="sales", table="results")
+            .build())
+
+        pipeline.run()
+        ```
+    """
+
+    def __init__(self, name: str) -> None:
+        """Initialize the pipeline builder.
+
+        Args:
+            name: The name of the pipeline.
+        """
+        self.name = name
+        self.steps: OrderedDict[str, PipelineStep] = OrderedDict()
+        self._step_counter = 0
+
+    def __getattr__(self, name: str) -> Callable[..., "PipelineBuilder"]:
+        """Dynamically create methods for pipeline actions.
+
+        This method is called when an attribute that doesn't exist is accessed.
+        It converts method calls like `read_files()` into the corresponding PipelineAction.
+
+        Args:
+            name: The method name being called.
+
+        Returns:
+            A callable that adds the corresponding pipeline step.
+
+        Raises:
+            AttributeError: If the method name doesn't correspond to a known action.
+        """
+        # Lazy import to avoid circular import issues
+        from .actions import pipeline_actions
+
+        # Convert method name to action name (e.g., read_files -> READ_FILES)
+        action_name = name.upper()
+
+        if action_name in pipeline_actions:
+            action_class = pipeline_actions[action_name]
+
+            def method(**kwargs: Any) -> "PipelineBuilder":
+                return self._add_step(action_class, **kwargs)
+
+            return method
+
+        raise AttributeError(
+            f"PipelineBuilder has no method '{name}'. Available actions: {list(pipeline_actions.keys())}"
+        )
+
+    def _add_step(self, action_class: type, step_name: str | None = None, **options: Any) -> Self:
+        """Add a step to the pipeline.
+
+        Args:
+            action_class: The PipelineAction class to instantiate.
+            step_name: Optional custom name for the step.
+            **options: Options to pass to the action.
+
+        Returns:
+            Self for method chaining.
+
+        Raises:
+            ValueError: If a step with the given name already exists.
+        """
+        if step_name is None:
+            step_name = f"step_{self._step_counter:03d}_{action_class.__name__}"
+
+        # Validate that step name is unique
+        if step_name in self.steps:
+            raise ValueError(
+                f"A step with name '{step_name}' already exists in the pipeline. "
+                f"Please provide a unique step_name. "
+                f"Existing steps: {list(self.steps.keys())}"
+            )
+
+        # Convert any PipelineBuilder instances in options to PipelineStep references
+        options = self._convert_builder_references(options)
+
+        # Set up context reference to previous step
+        context_ref = None
+        if self.steps:
+            context_ref = list(self.steps.keys())[-1]
+
+        step = PipelineStep(name=step_name, action=action_class(), options=options, _context_ref=context_ref)
+
+        # Remove any predecessors that are from already-executed external pipelines
+        # (these steps have results but aren't in our pipeline)
+        external_predecessors = set()
+        for pred_name in step._predecessors:
+            if pred_name not in self.steps and pred_name != context_ref:
+                # Check if this is a reference to an executed step from options
+                for opt_val in options.values():
+                    if isinstance(opt_val, PipelineStep) and opt_val.name == pred_name:
+                        # This is an external executed step, remove from predecessors
+                        external_predecessors.add(pred_name)
+                        break
+
+        step._predecessors -= external_predecessors
+
+        self.steps[step_name] = step
+        self._step_counter += 1
+        return self
+
+    def _convert_builder_references(self, options: dict[str, Any]) -> dict[str, Any]:
+        """Convert any PipelineBuilder instances in options to PipelineStep references.
+
+        This method recursively processes options to find PipelineBuilder instances and
+        converts them to their last step's PipelineStep reference. This allows users to
+        pass PipelineBuilder instances directly to actions that expect PipelineStep references.
+
+        Handles PipelineBuilder instances in:
+        - Direct values
+        - Lists
+        - Nested dictionaries
+
+        Args:
+            options: Dictionary of options that may contain PipelineBuilder instances.
+
+        Returns:
+            Dictionary with PipelineBuilder instances converted to PipelineStep references.
+
+        Raises:
+            ValueError: If a PipelineBuilder has no steps.
+        """
+        converted = {}
+        for key, value in options.items():
+            converted[key] = self._convert_value(value, key)
+        return converted
+
+    def _convert_value(self, value: Any, context: str = "") -> Any:
+        """Recursively convert a value, handling PipelineBuilder instances.
+
+        When a PipelineBuilder is passed as a value, it is executed immediately
+        and its last step is returned as the reference. This allows the pipeline
+        to be run before the main pipeline that references it.
+
+        Args:
+            value: The value to convert.
+            context: Context string for error messages (e.g., key name).
+
+        Returns:
+            The converted value.
+        """
+        if isinstance(value, PipelineBuilder):
+            # Build and run the referenced pipeline immediately
+            pipeline = value.build()
+            if not pipeline.steps:
+                context_msg = f" in '{context}'" if context else ""
+                raise ValueError(f"PipelineBuilder{context_msg} must have at least one step")
+
+            # Run the pipeline to populate the results
+            pipeline.run()
+
+            # Get the last step which now has results
+            last_step_name = list(pipeline.steps.keys())[-1]
+            last_step = pipeline.steps[last_step_name]
+
+            # Clear predecessors since this step is already executed and has its result
+            # This prevents the main pipeline from trying to resolve dependencies
+            # that don't exist in its own step dictionary
+            last_step._predecessors = set()
+            last_step._context_ref = None
+
+            return last_step
+        if isinstance(value, dict):
+            # Recursively convert nested dictionaries
+            return {k: self._convert_value(v, f"{context}.{k}" if context else k) for k, v in value.items()}
+        if isinstance(value, list):
+            # Recursively convert lists
+            return [
+                self._convert_value(item, f"{context}[{i}]" if context else f"[{i}]") for i, item in enumerate(value)
+            ]
+        return value
+
+    def build(self) -> Pipeline:
+        """Build the pipeline from the configured steps.
+
+        Returns:
+            A Pipeline object ready for execution.
+        """
+        return Pipeline(name=self.name, steps=self.steps)
+
+    def run(self) -> None:
+        """Build and run the pipeline immediately.
+
+        This is a convenience method equivalent to calling build().run().
+        """
+        pipeline = self.build()
+        pipeline.run()

cloe_nessy/pipeline/pipeline_config.py

@@ -83,6 +83,7 @@ class PipelineStepConfig(PipelineConfigBaseModel):
     context: str | None = None
     table_metadata: str | None = None
     options: dict = Field(default_factory=dict)
+    env: dict = Field(default_factory=dict)
 
 
 class PipelineConfig(PipelineConfigBaseModel):
@@ -90,3 +91,4 @@ class PipelineConfig(PipelineConfigBaseModel):
 
     name: str
     steps: OrderedDict[str, PipelineStepConfig]
+    env: dict[str, str] = Field(default_factory=dict)

cloe_nessy/pipeline/pipeline_context.py

@@ -1,6 +1,6 @@
 from typing import Any
 
-from pyspark.sql import DataFrame
+from cloe_nessy.session import DataFrame
 
 from ..models import Table
 

cloe_nessy/pipeline/pipeline_parsing_service.py

@@ -3,6 +3,7 @@ import re
 from collections import OrderedDict
 from enum import Enum
 from pathlib import Path
+from typing import Any
 
 import yaml
 
@@ -10,7 +11,7 @@ from ..logging import LoggerMixin
 from ..session import SessionManager
 from .actions import PipelineActionType, pipeline_actions
 from .pipeline import Pipeline
-from .pipeline_config import PipelineConfig
+from .pipeline_config import PipelineConfig, PipelineStepConfig
 from .pipeline_step import PipelineStep
 
 
@@ -63,49 +64,22 @@ class PipelineParsingService:
         if not yaml_str:
             raise ValueError("YAML content is empty.")
 
-        final_yaml_str = PipelineParsingService._replace_variables(yaml_str)
-        config = yaml.safe_load(final_yaml_str)
+        secrets_repl_yaml_str = PipelineParsingService._replace_secret_refs(yaml_str)
+        fixed_yaml_str = PipelineParsingService._fix_yaml_str_with_templates(secrets_repl_yaml_str)
+        config = yaml.safe_load(fixed_yaml_str)
         pipeline_config = PipelineConfig.metadata_to_instance(config)
-        steps = PipelineParsingService._get_steps(pipeline_config.steps)
+        steps = PipelineParsingService._get_steps(pipeline_config.steps, pipeline_config.env)
         pipeline = Pipeline(name=pipeline_config.name, steps=steps)  # type: ignore
         console_logger.info("Pipeline [ '%s' ] parsed successfully with %d steps.", pipeline.name, len(pipeline.steps))
         return pipeline
 
     @staticmethod
-    def _replace_variables(yaml_str: str) -> str:
-        """Replace variable placeholders in a YAML string.
-
-        Replaces environment variables with the pattern `{{env:var-name}}`. Where
-        the var-name is the name of the environment variable. Replaces secret
-        references with the pattern `{{secret-scope-name:secret-key}}`. Where
-        scope-name is the name of the secret scope and secret-key is the key of
-        the secret.
-
-        Args:
-            yaml_str: A string that can be parsed in YAML format.
-
-        Returns:
-            The same YAML string with environment variable placeholders replaced.
-        """
-        env_var_pattern = r"\{\{env:([^}]+)\}\}"
-        secret_ref_pattern = r"\{\{(?!step|env)([^}]+):([^}]+)\}\}"
-
-        def replace_with_env_var(match):
-            env_var_name = match.group(1)
-            env_var_value = os.getenv(env_var_name)
-            return env_var_value
-
-        def replace_with_secret(match):
-            secret_scope_name = match.group(1)
-            secret_key = match.group(2)
-            return SessionManager.get_utils().secrets.get(scope=secret_scope_name, key=secret_key)
-
-        env_replaced_yaml_string = re.sub(env_var_pattern, replace_with_env_var, yaml_str)
-        final_yaml_string = re.sub(secret_ref_pattern, replace_with_secret, env_replaced_yaml_string)
-        return final_yaml_string
-
-    @staticmethod
-    def _get_steps(step_configs, last_step_name: str | None = None):
+    def _get_steps(
+        step_configs: OrderedDict[str, PipelineStepConfig],
+        pipeline_env: dict[str, str],
+        last_step_name: str | None = None,
+    ) -> OrderedDict[str, PipelineStep]:
+        os_env = dict(os.environ)
         steps = OrderedDict()
         for step_name, step_config in step_configs.items():
             is_successor = step_config.is_successor
@@ -115,19 +89,99 @@ class PipelineParsingService:
             action = PipelineActionType[step_config.action.name].value()
             step = PipelineStep(
                 name=step_name,
+                env=step_config.env,
                 action=action,
                 options=step_config.options,
                 _context_ref=context_ref,
                 _table_metadata_ref=step_config.table_metadata,
             )
-            steps[step.name] = step
+            steps[step.name] = PipelineParsingService._resolve_env_vars(step, os_env, pipeline_env)
             last_step_name = step_name
         for step in steps.values():
             steps[step.name] = PipelineParsingService._replace_step_refs(steps, step)
         return steps
 
+    @staticmethod
+    def _replace_secret_refs(yaml_str: str) -> str:
+        """Replaces secret reference placeholders in a YAML string.
+
+        Replaces secret references with the pattern `{{secret-scope-name:secret-key}}`.
+        Where scope-name is the name of the secret scope and secret-key is the key of the secret.
+
+        Args:
+            yaml_str: A string that can be parsed in YAML format.
+
+        Returns:
+            The same YAML string with secret reference placeholders replaced.
+        """
+        secret_ref_pattern = r"\{\{(?!(?:env|step):)([^}]+):([^}]+)\}\}"
+
+        def replace_with_secret(match):
+            secret_scope_name = match.group(1)
+            secret_key = match.group(2)
+            return SessionManager.get_utils().secrets.get(scope=secret_scope_name, key=secret_key)
+
+        return re.sub(secret_ref_pattern, replace_with_secret, yaml_str)
+
+    @staticmethod
+    def _resolve_env_vars(step: PipelineStep, os_env: dict[str, str], pipeline_env: dict[str, str]) -> PipelineStep:
+        """Resolves environment variable placeholders in step definition.
+
+        Resolves environment variables with the pattern `{{env:var-name}}`,
+        where the `var-name` is the name of the environment variable.
+
+        Args:
+            step: Step definition, where replacement is occurred.
+            os_env: OS scope environment variable.
+            pipeline_env: Pipeline scope environment variables.
+
+        Returns:
+            The same step definition with environment variable placeholders replaced.
+
+        Raises:
+            KeyError: If the specified key is not found in the environment variables.
+        """
+        env_var_pattern = re.compile(r"\{\{env:([A-Z_][A-Z0-9_]*)\}\}")
+
+        def _resolve_object(obj: Any) -> Any:
+            if isinstance(obj, str):
+                return _resolve_string(obj)
+            if isinstance(obj, list):
+                return [_resolve_object(i) for i in obj]
+            if isinstance(obj, dict):
+                return {k: _resolve_object(v) for k, v in obj.items()}
+            return obj
+
+        def _resolve_string(value: str) -> str:
+            def repl(match):
+                key = match.group(1)
+                if key not in effective_env:
+                    raise KeyError(f"Environment variable '{key}' is not defined")
+                return str(effective_env[key])
+
+            return env_var_pattern.sub(repl, value)
+
+        if step.options:
+            effective_env = {**os_env, **pipeline_env, **step.env}
+            for option, value in step.options.items():
+                step.options[option] = _resolve_object(value)
+
+        return step
+
     @staticmethod
     def _replace_step_refs(steps: OrderedDict[str, PipelineStep], step: PipelineStep) -> PipelineStep:
+        """Replaces other steps reference placeholders in a step definition.
+
+        Replaces other steps references with the pattern `((step:step-name))`.
+        Where the `step-name` is the name of the referenced step.
+
+        Args:
+            steps: All pipeline steps definitions.
+            step: Step definition, where replacement is occurred.
+
+        Returns:
+            The same step definition with referenced step names replaced.
+        """
         step_ref_pattern = r"\(\(step:([^)]+)\)\)"
 
         def _handle_string_value(value: str, option: str):
@@ -154,3 +208,14 @@ class PipelineParsingService:
                     _handle_list_value(value, option)
 
         return step
+
+    @staticmethod
+    def _fix_yaml_str_with_templates(yaml_str: str) -> str:
+        """Fixes unquoted {{env:...}} templates before yaml.safe_load."""
+        unquoted_template = re.compile(r"(:)\s*(\{\{env:[^}]+\}\})(?=\s*$|\s+#)", re.MULTILINE)
+
+        def replacer(match):
+            colon, template = match.groups()
+            return f'{colon} "{template}"'
+
+        return unquoted_template.sub(replacer, yaml_str)

cloe_nessy/pipeline/pipeline_step.py

@@ -15,6 +15,7 @@ class PipelineStep:
     Attributes:
         name: The name of the step.
         action: The action to be executed.
+        env: The step environment variables.
         is_successor: A boolean indicating if the step is a successor and takes
             the previous steps context.
         context: The context of the step.
@@ -26,6 +27,7 @@ class PipelineStep:
 
     name: str
     action: PipelineAction
+    env: dict[str, str] = field(default_factory=lambda: {})
     context: PipelineContext = field(default_factory=lambda: PipelineContext())
     options: dict[str, Any] = field(default_factory=lambda: {})
     result: PipelineContext = field(default_factory=lambda: PipelineContext())

cloe_nessy/session/__init__.py

@@ -1,3 +1,4 @@
+from .pyspark_compat import DataFrame, SparkSession
 from .session_manager import SessionManager
 
-__all__ = ["SessionManager"]
+__all__ = ["SessionManager", "DataFrame", "SparkSession"]

cloe_nessy/session/pyspark_compat.py

@@ -0,0 +1,15 @@
+from typing import TYPE_CHECKING
+
+from pyspark.sql.utils import is_remote
+
+if TYPE_CHECKING:
+    from pyspark.sql import Column, DataFrame, SparkSession
+else:
+    # Real runtime imports
+    if is_remote():
+        from pyspark.sql.connect.dataframe import DataFrame
+        from pyspark.sql.connect.session import SparkSession
+    else:
+        from pyspark.sql import DataFrame, SparkSession
+
+__all__ = ["SparkSession", "DataFrame", "Column"]

cloe_nessy/session/session_manager.py

@@ -3,7 +3,7 @@ import os
 from enum import Enum
 from typing import Any
 
-from pyspark.sql import SparkSession
+from cloe_nessy.session import SparkSession
 
 from ..logging import LoggerMixin
 

{cloe_nessy-0.3.18.dist-info → cloe_nessy-1.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.18
+Version: 1.0.0
 Summary: Your friendly datalake monster.
 Project-URL: homepage, https://initions.com/
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
@@ -16,7 +16,7 @@ Requires-Python: <3.13,>=3.11
 Requires-Dist: azure-identity<2.0.0,>=1.19.0
 Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.8
 Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
-Requires-Dist: fsspec<2025.7.1,>=2025.7.0
+Requires-Dist: fsspec<2025.12.1,>=2025.12.0
 Requires-Dist: httpx<1.0.0,>=0.27.2
 Requires-Dist: jinja2<4.0.0,>=3.1.4
 Requires-Dist: matplotlib<4.0.0,>=3.9.2
@@ -58,12 +58,12 @@ Extract-Transform-Load (ETL) Workflow.
 
 When you are contributing, please refer to our Contribution Guide in the *nessy*
 Docs
-[here](https://yellow-mud-0b9177e03.2.azurestaticapps.net/tool_docs/nessy/Developer-Guide/)!
+[here](https://mango-tree-0b8dd3b03.1.azurestaticapps.net/tool_docs/nessy/Developer-Guide/)!
 
 ## Usage
 
 Please find the User Guide
-[here](https://yellow-mud-0b9177e03.2.azurestaticapps.net/tool_docs/nessy/User-Guide/)!
+[here](https://mango-tree-0b8dd3b03.1.azurestaticapps.net/tool_docs/nessy/User-Guide/)!
 
 ## Contact