cloe-nessy 1.0.10__py3-none-any.whl → 1.0.12__py3-none-any.whl

cloe_nessy/logging/logger_mixin.py

@@ -34,6 +34,7 @@ class LoggerMixin:
             logger_name=f"Console:{self.__class__.__name__}",
             logging_level=level if level is not None else logging_settings.log_level_console,
             log_format=log_format if log_format is not None else logging_settings.log_format_console,
+            datefmt="%Y-%m-%d %H:%M:%S",
         )
         return logger
 

cloe_nessy/models/adapter/unity_catalog_adapter.py

@@ -18,7 +18,7 @@ class UnityCatalogAdapter(LoggerMixin):
         """Initializes the UnityCatalogAdapter class."""
         self._spark = spark or SessionManager.get_spark_session()
         self._console_logger = self.get_console_logger()
-        self._catalogs = self.get_catalogs()
+        self._catalogs: list[Catalog] | None = None
 
     def _execute_sql(self, query):
         """Execute a SQL query and return a DataFrame.
@@ -57,6 +57,8 @@ class UnityCatalogAdapter(LoggerMixin):
         Returns:
             The Catalog with the specified name.
         """
+        if self._catalogs is None:
+            self._catalogs: list[Catalog] = self.get_catalogs()
         for catalog in self._catalogs:
             if catalog.name == name:
                 return catalog

cloe_nessy/pipeline/actions/write_delta_merge.py

@@ -106,6 +106,8 @@ class WriteDeltaMergeAction(PipelineAction):
             raise ValueError("Data is required for the merge operation.")
         if context.table_metadata is None and table_identifier is None:
             raise ValueError("Table metadata or a table identifier are required for the merge operation.")
+        if context.table_metadata is not None and table_identifier is not None:
+            raise ValueError("Provide either table metadata or a table identifier, not both.")
 
         if table_identifier is not None:
             context.table_metadata = UnityCatalogAdapter().get_table_by_name(table_identifier)

cloe_nessy/pipeline/pipeline.py

@@ -56,13 +56,13 @@ class Pipeline(LoggerMixin):
             step.context.table_metadata = self.steps[step._table_metadata_ref].result.table_metadata
 
         try:
-            self._console_logger.info(f"Starting execution of step: {step.name}")
+            self._console_logger.info(f">>> Starting execution of step: {step.name}")
             step.run()
         except Exception as err:
-            self._console_logger.error(f"Execution of step {step.name} failed with error: {str(err)}")
+            self._console_logger.error(f">>> Execution of step {step.name} failed with error: {str(err)}")
             raise err
         else:
-            self._console_logger.info(f"Execution of step {step.name} succeeded.")
+            self._console_logger.info(f">>> Execution of step {step.name} succeeded.")
 
     def _get_ready_to_run_steps(self, remaining_steps: list[str], g: nx.DiGraph) -> set[str]:
         """Identifies and returns the steps that are ready to run.

cloe_nessy/pipeline/pipeline_parsing_service.py

@@ -67,6 +67,7 @@ class PipelineParsingService:
         try:
             secrets_repl_yaml_str = PipelineParsingService._replace_secret_refs(yaml_str)
             fixed_yaml_str = PipelineParsingService._fix_yaml_str_with_templates(secrets_repl_yaml_str)
+            PipelineParsingService._validate_yaml(fixed_yaml_str)
             config = yaml.safe_load(fixed_yaml_str)
         except yaml.YAMLError as e:
             console_logger.error("Failed to parse YAML. Check for syntax errors or unquoted template variables.")
@@ -78,9 +79,49 @@ class PipelineParsingService:
         pipeline_config = PipelineConfig.metadata_to_instance(config)
         steps = PipelineParsingService._get_steps(pipeline_config.steps, pipeline_config.env)
         pipeline = Pipeline(name=pipeline_config.name, steps=steps)  # type: ignore
-        console_logger.info("Pipeline [ '%s' ] parsed successfully with %d steps.", pipeline.name, len(pipeline.steps))
+        console_logger.info(
+            "Pipeline [ '%s' ] parsed successfully with %d steps.",
+            pipeline.name,
+            len(pipeline.steps),
+        )
         return pipeline
 
+    @staticmethod
+    def _validate_yaml(yaml_str: str) -> None:
+        """Validates YAML string including checking for duplicate keys.
+
+        Args:
+            yaml_str: A string that can be parsed in YAML format.
+
+        Raises:
+            yaml.YAMLError: If the YAML is invalid.
+            ValueError: If duplicate keys are found.
+        """
+
+        def no_duplicates_constructor(loader, node, deep=False):
+            """Check for duplicate keys in YAML mappings."""
+            mapping = {}
+            for key_node, value_node in node.value:
+                key = loader.construct_object(key_node, deep=deep)
+                if key in mapping:
+                    raise ValueError(f"Duplicate key found in YAML: '{key}'")
+                value = loader.construct_object(value_node, deep=deep)
+                mapping[key] = value
+            return mapping
+
+        class DuplicateCheckLoader(yaml.SafeLoader):
+            pass
+
+        DuplicateCheckLoader.add_constructor(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+            no_duplicates_constructor,
+        )
+
+        try:
+            yaml.load(yaml_str, Loader=DuplicateCheckLoader)
+        except yaml.YAMLError as e:
+            raise yaml.YAMLError(f"Invalid YAML: {e}") from e
+
     @staticmethod
     def _get_steps(
         step_configs: OrderedDict[str, PipelineStepConfig],

cloe_nessy/pipeline/pipeline_step.py

@@ -36,6 +36,7 @@ class PipelineStep:
     _table_metadata_ref: str | None = None
 
     def __post_init__(self) -> None:
+        self._validate_step_name()
         if not isinstance(self.action, PipelineAction):
             raise ValueError("action must be a PipelineAction subclass.")
         if self._context_ref:
@@ -47,6 +48,22 @@ class PipelineStep:
                 if isinstance(val, PipelineStep):
                     self._predecessors.add(val.name)
 
+    def _validate_step_name(self) -> None:
+        """Validates that the step name does not contain forbidden characters.
+
+        These character have lead to parsing issues when using joins and unions
+        in the past.
+
+        Raises:
+            ValueError: If step name contains brackets or parentheses.
+        """
+        invalid_chars = ["[", "]", "(", ")", "{", "}"]
+        for char in invalid_chars:
+            if char in self.name:
+                raise ValueError(
+                    f"Invalid step name '{self.name}': step names cannot contain brackets or parentheses ({', '.join(invalid_chars)})"
+                )
+
     def run(self) -> None:
         """Execute the action on the context."""
         self.result = self.action.run(context=self.context, **self.options)

{cloe_nessy-1.0.10.dist-info → cloe_nessy-1.0.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 1.0.10
+Version: 1.0.12
 Summary: Your friendly datalake monster.
 Project-URL: homepage, https://initions.com/
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: <3.14,>=3.11
 Requires-Dist: azure-identity<2.0.0,>=1.19.0
-Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.8
+Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.11
 Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
 Requires-Dist: fsspec<2026.1.1,>=2026.1.0
 Requires-Dist: httpx<1.0.0,>=0.27.2

{cloe_nessy-1.0.10.dist-info → cloe_nessy-1.0.12.dist-info}/RECORD

@@ -45,7 +45,7 @@ cloe_nessy/integration/writer/delta_writer/delta_table_operation_type.py,sha256=
 cloe_nessy/integration/writer/delta_writer/delta_writer_base.py,sha256=B7PwPHKrsJL0ZxBT-H9wWSy0gn7shqNDJ0AbrpMHyMg,10135
 cloe_nessy/integration/writer/delta_writer/exceptions.py,sha256=xPmGiYV0xQXauln5Oh34E5vbm0rVcs6xCh-SJSb2bw0,107
 cloe_nessy/logging/__init__.py,sha256=ySVCVbdyR3Dno_tl2ZfiER_7EVaDoQMHVkNyfdMZumY,65
-cloe_nessy/logging/logger_mixin.py,sha256=xA12LIVn4yUEoaKmm7nGC1-U3ddSo_HiL3I5MfkvEwU,7409
+cloe_nessy/logging/logger_mixin.py,sha256=Ws3sirAi7CVyWWc_ClQs0wzdMIJ1LNNrMMkFlO1V0dw,7450
 cloe_nessy/models/__init__.py,sha256=-FmWEJ1Oq1njSopjc0R7GmT64mLSmALkm8PkHNzy9Y8,327
 cloe_nessy/models/catalog.py,sha256=ayC1sMp4cNLAZtu0ICVV3Us6-o4hn8U9tpzzvxC9RAs,177
 cloe_nessy/models/column.py,sha256=W4V1Ls1d60VyZ1Ko9Yu9eSipcMbxSzKicn0aloHPiR0,2027
@@ -56,7 +56,7 @@ cloe_nessy/models/table.py,sha256=-MQLwKthcO4JE99lbikJL1cnt27aEZmt3o96R9CtoTU,12
 cloe_nessy/models/types.py,sha256=XRbuJGdTNa6aXyE3IAzs_J9gVjbfkzMDLfGl-k6jI_4,223
 cloe_nessy/models/volume.py,sha256=BPEy6DW0Yig-mFQiNgm9P5JoO6Vcvtxcipnk2-H8Q_k,2456
 cloe_nessy/models/adapter/__init__.py,sha256=m36W_mqwB3dCYnCIt0fLOSHS4E1VU8FRGoaum4Gf95o,90
-cloe_nessy/models/adapter/unity_catalog_adapter.py,sha256=a-14Ys-AevVYQd0xeJU1syLxjT5Wzo4uog1hFSEs76M,12651
+cloe_nessy/models/adapter/unity_catalog_adapter.py,sha256=75QNiYgvWGG2zhxslNyABIZRuKtgTP7DlYrF5PdfsWQ,12757
 cloe_nessy/models/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/models/mixins/read_instance_mixin.py,sha256=HT42qor6IltR-BTfe_DYa5ylntE7qbArs-3cSjo4KXU,4649
 cloe_nessy/models/mixins/template_loader_mixin.py,sha256=5MXhEGBFlq3dwZvINEyBowSlipNnVun2H_TmhI_fsS4,549
@@ -66,14 +66,14 @@ cloe_nessy/object_manager/__init__.py,sha256=3sle0vNpPwBOkycxA3XVS9m4XZf5LD3Qd4N
 cloe_nessy/object_manager/table_manager.py,sha256=M4JWgga7A8MHll5QJ42TUU5X4eUrQzhbwaZtVBgoQPY,13902
 cloe_nessy/object_manager/volume_manager.py,sha256=6epd3KXzcNH04EvaKubAfLsaUm9qBMeT3KNvMK04gGs,2727
 cloe_nessy/pipeline/__init__.py,sha256=BUzL4HJaCXWmK7OgKaxdwK72JrrdzfzIvyxOGtM28U0,417
-cloe_nessy/pipeline/pipeline.py,sha256=L4wk3b06LNWRj01nnAkuQpeRrwFTyaV1xTpgYAg4sak,10819
+cloe_nessy/pipeline/pipeline.py,sha256=-JcdRL8PV1Leo0984Z2jOJ4EL2nuMwnqX4rIwu4JxEg,10831
 cloe_nessy/pipeline/pipeline_action.py,sha256=S7IVFdmG12fRBzHuE_DiWn7qlMtApz6IloVd2Fj31Sg,1944
 cloe_nessy/pipeline/pipeline_builder.py,sha256=_BBl43two0pherkTXZ-Yrpt6XcLW8Q-Z98qxbFIsMao,7929
 cloe_nessy/pipeline/pipeline_config.py,sha256=oVQ-IH4etTGZVVEnE-5iDPLYOtWpvDlltWFv1nevnqQ,3229
 cloe_nessy/pipeline/pipeline_context.py,sha256=eCOcjyE16rGRom3L85Gy_BbncfQD6i1x31yrWqZws-4,1881
-cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=qVDckjEqv4qYD0jUABs4BSbgYC3w-A_SpqBmqr7qCxk,9978
+cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=XJu2v4qzSBMuLRZs35CIabLPQHY3zV42Vurf4VuN9v0,11363
 cloe_nessy/pipeline/pipeline_plotting_service.py,sha256=OjdYDQW19yXjdcqs7bDMlyWmv0cQz7Qn4I2cakBgN2E,13139
-cloe_nessy/pipeline/pipeline_step.py,sha256=oTnlvRpB0fbOBQXbPe1URstA5fv-97igCHt_41fKCAk,2082
+cloe_nessy/pipeline/pipeline_step.py,sha256=ui3hDOAuuX-j62gg6o8RogHzCjZHNsTYWpLZwO76VhE,2753
 cloe_nessy/pipeline/actions/__init__.py,sha256=FfAnSIl-0T6pnaWhClkDqV8nfTdvLvZZJdwycsZMLPw,2990
 cloe_nessy/pipeline/actions/read_api.py,sha256=YMOWPCyxitU5v6HHH_AZqpbHQenPU4-WlaOrb-NsIIk,16245
 cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=MK67NIB5qXTDwCC0EFwHYoOkelFnwY6Z4REpC5BlFb4,6359
@@ -101,7 +101,7 @@ cloe_nessy/pipeline/actions/transform_union.py,sha256=SZtEzh567CIExUj9yMEgshE28h
 cloe_nessy/pipeline/actions/transform_with_column.py,sha256=5EG_H4MTiQVCfgkjfyof4qqw9xB3MJh-v0lsoYlbVCU,4967
 cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=rYK_V08D8w4CBClRBGwj18amhZOVmP6VTTO-xqy7We8,4849
 cloe_nessy/pipeline/actions/write_delta_append.py,sha256=e1g4mDhwAZdKyt4Gb7ZzHcQrJ1duSl8qOn6ONizRsoM,2934
-cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=5Svi9Z7xg2Ey0WPq4MzS509vMDOM7mDGUa5OBaWbjkc,6438
+cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=ndJf9n3VP8YhTSVLgyURnbtOmC5i3YfVQWtfRjBV1j0,6613
 cloe_nessy/pipeline/actions/write_file.py,sha256=JZ8UZslxUn_ttYt5wDyvtHFq2FqYk3vOR8kvExJI8pk,3212
 cloe_nessy/pipeline/utils/__init__.py,sha256=xi02UjBMiXWD7b9gDvww4gyRyowb0eRd_6Wbu0F_cro,118
 cloe_nessy/pipeline/utils/delta_load_utils.py,sha256=KitMNruxePEkecI0h4Jint1JwJpaEog5mCOchMkgan8,1495
@@ -113,6 +113,6 @@ cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_Up
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/column_names.py,sha256=dCNtm61mc5aLkY2oE4rlfN3VLCrpot6fOESjAZmCmhA,361
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-1.0.10.dist-info/METADATA,sha256=NaIr91k5JZzPptDRnVq28odbhx2bhVxDuixXGbL3QIc,3290
-cloe_nessy-1.0.10.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-cloe_nessy-1.0.10.dist-info/RECORD,,
+cloe_nessy-1.0.12.dist-info/METADATA,sha256=W3Gs63IM-gG61xJVlCgs7R3FB9VcI94VRdaFfElczUg,3291
+cloe_nessy-1.0.12.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+cloe_nessy-1.0.12.dist-info/RECORD,,