cloe-nessy 1.0.9__py3-none-any.whl → 1.0.11__py3-none-any.whl

cloe_nessy/integration/delta_loader/delta_loader.py

@@ -39,7 +39,10 @@ class DeltaLoader(ABC, LoggerMixin):
             or f"{self.table_identifier.split('.')[0]}.{self.table_identifier.split('.')[1]}.metadata_delta_load"
         )
         table_manager = TableManager()
-        table_manager.create_table(table=DeltaLoaderMetadataTable(identifier=self.metadata_table_identifier))
+        table_manager.create_table(
+            table=DeltaLoaderMetadataTable(identifier=self.metadata_table_identifier),
+            ignore_if_exists=True,
+        )
 
     @abstractmethod
     def read_data(

cloe_nessy/models/adapter/unity_catalog_adapter.py

@@ -18,7 +18,7 @@ class UnityCatalogAdapter(LoggerMixin):
         """Initializes the UnityCatalogAdapter class."""
         self._spark = spark or SessionManager.get_spark_session()
         self._console_logger = self.get_console_logger()
-        self._catalogs = self.get_catalogs()
+        self._catalogs: list[Catalog] | None = None
 
     def _execute_sql(self, query):
         """Execute a SQL query and return a DataFrame.
@@ -57,6 +57,8 @@ class UnityCatalogAdapter(LoggerMixin):
         Returns:
             The Catalog with the specified name.
         """
+        if self._catalogs is None:
+            self._catalogs: list[Catalog] = self.get_catalogs()
         for catalog in self._catalogs:
             if catalog.name == name:
                 return catalog

cloe_nessy/pipeline/actions/write_delta_merge.py

@@ -106,6 +106,8 @@ class WriteDeltaMergeAction(PipelineAction):
             raise ValueError("Data is required for the merge operation.")
         if context.table_metadata is None and table_identifier is None:
             raise ValueError("Table metadata or a table identifier are required for the merge operation.")
+        if context.table_metadata is not None and table_identifier is not None:
+            raise ValueError("Provide either table metadata or a table identifier, not both.")
 
         if table_identifier is not None:
             context.table_metadata = UnityCatalogAdapter().get_table_by_name(table_identifier)

cloe_nessy/pipeline/pipeline_parsing_service.py

@@ -64,15 +64,64 @@ class PipelineParsingService:
         if not yaml_str:
             raise ValueError("YAML content is empty.")
 
-        secrets_repl_yaml_str = PipelineParsingService._replace_secret_refs(yaml_str)
-        fixed_yaml_str = PipelineParsingService._fix_yaml_str_with_templates(secrets_repl_yaml_str)
-        config = yaml.safe_load(fixed_yaml_str)
+        try:
+            secrets_repl_yaml_str = PipelineParsingService._replace_secret_refs(yaml_str)
+            fixed_yaml_str = PipelineParsingService._fix_yaml_str_with_templates(secrets_repl_yaml_str)
+            PipelineParsingService._validate_yaml(fixed_yaml_str)
+            config = yaml.safe_load(fixed_yaml_str)
+        except yaml.YAMLError as e:
+            console_logger.error("Failed to parse YAML. Check for syntax errors or unquoted template variables.")
+            console_logger.error("Original error: %s", str(e))
+            raise ValueError(
+                f"YAML parsing failed. Ensure all template variables like {{env:VAR}} are "
+                f"properly formatted and YAML syntax is correct. Error: {e}"
+            ) from e
         pipeline_config = PipelineConfig.metadata_to_instance(config)
         steps = PipelineParsingService._get_steps(pipeline_config.steps, pipeline_config.env)
         pipeline = Pipeline(name=pipeline_config.name, steps=steps)  # type: ignore
-        console_logger.info("Pipeline [ '%s' ] parsed successfully with %d steps.", pipeline.name, len(pipeline.steps))
+        console_logger.info(
+            "Pipeline [ '%s' ] parsed successfully with %d steps.",
+            pipeline.name,
+            len(pipeline.steps),
+        )
         return pipeline
 
+    @staticmethod
+    def _validate_yaml(yaml_str: str) -> None:
+        """Validates YAML string including checking for duplicate keys.
+
+        Args:
+            yaml_str: A string that can be parsed in YAML format.
+
+        Raises:
+            yaml.YAMLError: If the YAML is invalid.
+            ValueError: If duplicate keys are found.
+        """
+
+        def no_duplicates_constructor(loader, node, deep=False):
+            """Check for duplicate keys in YAML mappings."""
+            mapping = {}
+            for key_node, value_node in node.value:
+                key = loader.construct_object(key_node, deep=deep)
+                if key in mapping:
+                    raise ValueError(f"Duplicate key found in YAML: '{key}'")
+                value = loader.construct_object(value_node, deep=deep)
+                mapping[key] = value
+            return mapping
+
+        class DuplicateCheckLoader(yaml.SafeLoader):
+            pass
+
+        DuplicateCheckLoader.add_constructor(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+            no_duplicates_constructor,
+        )
+
+        try:
+            yaml.load(yaml_str, Loader=DuplicateCheckLoader)
+        except yaml.YAMLError as e:
+            raise yaml.YAMLError(f"Invalid YAML: {e}") from e
+
     @staticmethod
     def _get_steps(
         step_configs: OrderedDict[str, PipelineStepConfig],
@@ -211,11 +260,24 @@ class PipelineParsingService:
 
     @staticmethod
     def _fix_yaml_str_with_templates(yaml_str: str) -> str:
-        """Fixes unquoted {{env:...}} templates (with optional prefix/suffix) before yaml.safe_load. Skips values already quoted with ' or "."""
-        unquoted_template = re.compile(r"(:)\s*(?!['\"])([^\s#]*\{\{env:[^}]+\}\}[^\s#]*)", re.MULTILINE)
+        """Fixes unquoted template patterns before yaml.safe_load.
+
+        Handles:
+        - {{env:...}} - environment variables
+        - ((step:...)) - step references
+
+        Skips values already quoted with ' or ".
+        """
+        # Match any unquoted template pattern (env or step)
+        # This regex matches: colon + optional whitespace + unquoted value containing template
+        unquoted_template = re.compile(
+            r"(:)\s*(?!['\"])([^\s#]*(?:\{\{env:[^}]+\}\}|\(\(step:[^)]+\)\))[^\s#]*)", re.MULTILINE
+        )
 
         def replacer(match):
             colon, template = match.groups()
-            return f'{colon} "{template}"'
+            # Escape any double quotes in the template value
+            escaped_template = template.replace('"', '\\"')
+            return f'{colon} "{escaped_template}"'
 
         return unquoted_template.sub(replacer, yaml_str)

cloe_nessy/pipeline/pipeline_step.py

@@ -36,6 +36,7 @@ class PipelineStep:
     _table_metadata_ref: str | None = None
 
     def __post_init__(self) -> None:
+        self._validate_step_name()
         if not isinstance(self.action, PipelineAction):
             raise ValueError("action must be a PipelineAction subclass.")
         if self._context_ref:
@@ -47,6 +48,22 @@ class PipelineStep:
                 if isinstance(val, PipelineStep):
                     self._predecessors.add(val.name)
 
+    def _validate_step_name(self) -> None:
+        """Validates that the step name does not contain forbidden characters.
+
+        These character have lead to parsing issues when using joins and unions
+        in the past.
+
+        Raises:
+            ValueError: If step name contains brackets or parentheses.
+        """
+        invalid_chars = ["[", "]", "(", ")", "{", "}"]
+        for char in invalid_chars:
+            if char in self.name:
+                raise ValueError(
+                    f"Invalid step name '{self.name}': step names cannot contain brackets or parentheses ({', '.join(invalid_chars)})"
+                )
+
     def run(self) -> None:
         """Execute the action on the context."""
         self.result = self.action.run(context=self.context, **self.options)

{cloe_nessy-1.0.9.dist-info → cloe_nessy-1.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 1.0.9
+Version: 1.0.11
 Summary: Your friendly datalake monster.
 Project-URL: homepage, https://initions.com/
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>

{cloe_nessy-1.0.9.dist-info → cloe_nessy-1.0.11.dist-info}/RECORD

@@ -21,7 +21,7 @@ cloe_nessy/file_utilities/strategies/utils_strategy.py,sha256=w4nrS6IcPPN7UBFBws
 cloe_nessy/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/integration/delta_loader/__init__.py,sha256=ZdBDde1uPtTCL_KAhilVmtVmmGvH5dHb05QsOozkteE,438
 cloe_nessy/integration/delta_loader/delta_load_options.py,sha256=W3P3gwtkBW3pWbUEOK6agvJfj7QivXhEr_s0dwxhJxQ,1369
-cloe_nessy/integration/delta_loader/delta_loader.py,sha256=RiHCsCc5_70KVYT2tiZFbuJyb6xPKNtD1ExEtwLO-1c,6946
+cloe_nessy/integration/delta_loader/delta_loader.py,sha256=2uSsfFpL3R_Qovx-bOYnhWh3x48r3Otp9xFv-gyoZy8,7004
 cloe_nessy/integration/delta_loader/delta_loader_factory.py,sha256=vB1cL6-Nc3SkLH1xtazMbMF1MnNYq8-g3GHZzRE3QmE,2251
 cloe_nessy/integration/delta_loader/delta_loader_metadata_table.py,sha256=G_EWUY76ZlbsPZB9LCGlOLVezk7DK6peYXEgt7-sTQE,1683
 cloe_nessy/integration/delta_loader/strategies/__init__.py,sha256=1o5fRWenL5KnUg1hf7kmTuTpG9pbMxchiQTub52Qvwo,255
@@ -56,7 +56,7 @@ cloe_nessy/models/table.py,sha256=-MQLwKthcO4JE99lbikJL1cnt27aEZmt3o96R9CtoTU,12
 cloe_nessy/models/types.py,sha256=XRbuJGdTNa6aXyE3IAzs_J9gVjbfkzMDLfGl-k6jI_4,223
 cloe_nessy/models/volume.py,sha256=BPEy6DW0Yig-mFQiNgm9P5JoO6Vcvtxcipnk2-H8Q_k,2456
 cloe_nessy/models/adapter/__init__.py,sha256=m36W_mqwB3dCYnCIt0fLOSHS4E1VU8FRGoaum4Gf95o,90
-cloe_nessy/models/adapter/unity_catalog_adapter.py,sha256=a-14Ys-AevVYQd0xeJU1syLxjT5Wzo4uog1hFSEs76M,12651
+cloe_nessy/models/adapter/unity_catalog_adapter.py,sha256=75QNiYgvWGG2zhxslNyABIZRuKtgTP7DlYrF5PdfsWQ,12757
 cloe_nessy/models/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/models/mixins/read_instance_mixin.py,sha256=HT42qor6IltR-BTfe_DYa5ylntE7qbArs-3cSjo4KXU,4649
 cloe_nessy/models/mixins/template_loader_mixin.py,sha256=5MXhEGBFlq3dwZvINEyBowSlipNnVun2H_TmhI_fsS4,549
@@ -71,9 +71,9 @@ cloe_nessy/pipeline/pipeline_action.py,sha256=S7IVFdmG12fRBzHuE_DiWn7qlMtApz6Ilo
 cloe_nessy/pipeline/pipeline_builder.py,sha256=_BBl43two0pherkTXZ-Yrpt6XcLW8Q-Z98qxbFIsMao,7929
 cloe_nessy/pipeline/pipeline_config.py,sha256=oVQ-IH4etTGZVVEnE-5iDPLYOtWpvDlltWFv1nevnqQ,3229
 cloe_nessy/pipeline/pipeline_context.py,sha256=eCOcjyE16rGRom3L85Gy_BbncfQD6i1x31yrWqZws-4,1881
-cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=sMnUDAL0PBLbKupC_TWxBT1X9-Y9I5dPSqAxAkWyweA,9102
+cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=XJu2v4qzSBMuLRZs35CIabLPQHY3zV42Vurf4VuN9v0,11363
 cloe_nessy/pipeline/pipeline_plotting_service.py,sha256=OjdYDQW19yXjdcqs7bDMlyWmv0cQz7Qn4I2cakBgN2E,13139
-cloe_nessy/pipeline/pipeline_step.py,sha256=oTnlvRpB0fbOBQXbPe1URstA5fv-97igCHt_41fKCAk,2082
+cloe_nessy/pipeline/pipeline_step.py,sha256=ui3hDOAuuX-j62gg6o8RogHzCjZHNsTYWpLZwO76VhE,2753
 cloe_nessy/pipeline/actions/__init__.py,sha256=FfAnSIl-0T6pnaWhClkDqV8nfTdvLvZZJdwycsZMLPw,2990
 cloe_nessy/pipeline/actions/read_api.py,sha256=YMOWPCyxitU5v6HHH_AZqpbHQenPU4-WlaOrb-NsIIk,16245
 cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=MK67NIB5qXTDwCC0EFwHYoOkelFnwY6Z4REpC5BlFb4,6359
@@ -101,7 +101,7 @@ cloe_nessy/pipeline/actions/transform_union.py,sha256=SZtEzh567CIExUj9yMEgshE28h
 cloe_nessy/pipeline/actions/transform_with_column.py,sha256=5EG_H4MTiQVCfgkjfyof4qqw9xB3MJh-v0lsoYlbVCU,4967
 cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=rYK_V08D8w4CBClRBGwj18amhZOVmP6VTTO-xqy7We8,4849
 cloe_nessy/pipeline/actions/write_delta_append.py,sha256=e1g4mDhwAZdKyt4Gb7ZzHcQrJ1duSl8qOn6ONizRsoM,2934
-cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=5Svi9Z7xg2Ey0WPq4MzS509vMDOM7mDGUa5OBaWbjkc,6438
+cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=ndJf9n3VP8YhTSVLgyURnbtOmC5i3YfVQWtfRjBV1j0,6613
 cloe_nessy/pipeline/actions/write_file.py,sha256=JZ8UZslxUn_ttYt5wDyvtHFq2FqYk3vOR8kvExJI8pk,3212
 cloe_nessy/pipeline/utils/__init__.py,sha256=xi02UjBMiXWD7b9gDvww4gyRyowb0eRd_6Wbu0F_cro,118
 cloe_nessy/pipeline/utils/delta_load_utils.py,sha256=KitMNruxePEkecI0h4Jint1JwJpaEog5mCOchMkgan8,1495
@@ -113,6 +113,6 @@ cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_Up
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/column_names.py,sha256=dCNtm61mc5aLkY2oE4rlfN3VLCrpot6fOESjAZmCmhA,361
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-1.0.9.dist-info/METADATA,sha256=imtWtMJfxQToZnS30gWkb_DW3XIahzqkbVjYk3cajVA,3289
-cloe_nessy-1.0.9.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-cloe_nessy-1.0.9.dist-info/RECORD,,
+cloe_nessy-1.0.11.dist-info/METADATA,sha256=cvl6CZwjaIn9-vbsnDsX3znHZmlzFga_DE8W_qntpek,3290
+cloe_nessy-1.0.11.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+cloe_nessy-1.0.11.dist-info/RECORD,,