cloe-nessy 0.3.13.3b0__py3-none-any.whl → 0.3.13.4b0__py3-none-any.whl

cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py

@@ -197,7 +197,9 @@ class DeltaMergeWriter(BaseDeltaWriter):
         config = DeltaMergeConfig(dataframe_columns=data_frame.columns, **kwargs)
 
         delta_table = self.table_manager.get_delta_table(
-            table=table, location=storage_path, spark=data_frame.sparkSession
+            table=table,
+            location=storage_path,
+            spark=data_frame.sparkSession,
         )
 
         match_conditions = self._build_match_conditions(data_frame, config)

cloe_nessy/pipeline/actions/read_metadata_yaml.py

@@ -1,66 +1,94 @@
-import pathlib
+from pathlib import Path
 from typing import Any
 
-from ...models import Schema
+from ...models import Table
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
 
 
 class ReadMetadataYAMLAction(PipelineAction):
-    """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
+    """Reads table metadata from a yaml file using the [`Table`][cloe_nessy.models.table] model.
 
     Example:
-        ```yaml
-        Read Schema Metadata:
-            action: READ_METADATA_YAML_ACTION
-            options:
-                path: excel_file_folder/excel_files_june/
-                file_name: sales_schema.yml
-                table_name: sales
-        ```
+        === "Managed Table"
+            ```yaml
+            Read Table Metadata:
+                action: READ_METADATA_YAML_ACTION
+                options:
+                    file_path: metadata/schemas/bronze/sales_table.yml
+                    catalog_name: production
+                    schema_name: sales_data
+            ```
+        === "External Table"
+            ```yaml
+            Read Table Metadata:
+                action: READ_METADATA_YAML_ACTION
+                options:
+                    file_path: metadata/schemas/bronze/sales_table.yml
+                    catalog_name: production
+                    schema_name: sales_data
+                    storage_path: abfs://external_storage/sales_data/sales_table
+            ```
     """
 
     name: str = "READ_METADATA_YAML_ACTION"
 
-    @staticmethod
     def run(
+        self,
         context: PipelineContext,
         *,
-        path: str | None = None,
-        file_name: str | None = None,
-        table_name: str | None = None,
+        file_path: str | None = None,
+        catalog_name: str | None = None,
+        schema_name: str | None = None,
+        storage_path: str | None = None,
         **_: Any,
     ) -> PipelineContext:
-        """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
+        """Reads table metadata from a yaml file using the [`Table`][cloe_nessy.models.table] model.
 
         Args:
             context: The context in which this Action is executed.
-            path: The path to the data contract directory.
-            file_name: The name of the file that defines the schema.
-            table_name: The name of the table for which to retrieve metadata.
+            file_path: The path to the file that defines the table.
+            catalog_name: The name of the catalog for the table.
+            schema_name: The name of the schema for the table.
+            storage_path: The storage path for the table, if applicable. If not
+                provided, the table will be considered a managed table.
 
         Raises:
-            ValueError: If any issues occur while reading the schema, such as an invalid schema,
-                missing file, or missing path.
+            ValueError: If any issues occur while reading the table metadata, such as an invalid table,
+                missing file, missing path, or missing catalog/schema names.
 
         Returns:
             The context after the execution of this Action, containing the table metadata.
         """
-        if not path:
-            raise ValueError("No path provided. Please specify path to schema metadata.")
-        if not file_name:
-            raise ValueError("No file_name provided. Please specify file name.")
-        if not table_name:
-            raise ValueError("No table_name provided. Please specify table name.")
+        missing_params = []
+        if not file_path:
+            missing_params.append("file_path")
+        if not catalog_name:
+            missing_params.append("catalog_name")
+        if not schema_name:
+            missing_params.append("schema_name")
 
-        path_obj = pathlib.Path(path)
+        if missing_params:
+            raise ValueError(
+                f"Missing required parameters: {', '.join(missing_params)}. Please specify all required parameters."
+            )
 
-        schema, errors = Schema.read_instance_from_file(path_obj / file_name)
+        final_file_path = Path(file_path) if file_path else Path()
+
+        table, errors = Table.read_instance_from_file(
+            final_file_path,
+            catalog_name=catalog_name,
+            schema_name=schema_name,
+        )
         if errors:
-            raise ValueError(f"Errors while reading schema metadata: {errors}")
-        if not schema:
-            raise ValueError("No schema found in metadata.")
+            raise ValueError(f"Errors while reading table metadata: {errors}")
+        if not table:
+            raise ValueError("No table found in metadata.")
 
-        table = schema.get_table_by_name(table_name=table_name)
+        if not table.storage_path and storage_path:
+            self._console_logger.info(f"Setting storage path for table [ '{table.name}' ] to [ '{storage_path}' ]")
+            table.storage_path = storage_path
+            table.is_external = True
 
+        self._console_logger.info(f"Table [ '{table.name}' ] metadata read successfully from [ '{file_path}' ]")
         return context.from_existing(table_metadata=table)

{cloe_nessy-0.3.13.3b0.dist-info → cloe_nessy-0.3.13.4b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.13.3b0
+Version: 0.3.13.4b0
 Summary: Your friendly datalake monster.
 Project-URL: homepage, https://initions.com/
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>

{cloe_nessy-0.3.13.3b0.dist-info → cloe_nessy-0.3.13.4b0.dist-info}/RECORD

@@ -30,7 +30,7 @@ cloe_nessy/integration/writer/file_writer.py,sha256=SUDbN13ZzDhbM8DpOGFgM_Gkg70T
 cloe_nessy/integration/writer/writer.py,sha256=elFPLFrWR-qVE9qnBtzzzhyRALLQcRVuOsPS0rNmRt4,1741
 cloe_nessy/integration/writer/delta_writer/__init__.py,sha256=h2CT6Hllmk0nodlek27uqwniCzVZKMkYcPGyG9K2Z24,164
 cloe_nessy/integration/writer/delta_writer/delta_append_writer.py,sha256=TbpW-j87_H9dcUza34uR6VWslJez406y3_5N1ip0SnM,4740
-cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=3LQsrPNq9Xi71NLFqbB1Qk6tcheNPoQX3ngvCuIKwaw,10147
+cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=zhqPIPfAJTzSLFgBUCwFesUW7CcF1zCPRU-N_8yYjok,10172
 cloe_nessy/integration/writer/delta_writer/delta_table_operation_type.py,sha256=kiacqQ2FYQSzakJqZ9-ZHH3os4X7--QuER_2xx9y21k,971
 cloe_nessy/integration/writer/delta_writer/delta_writer_base.py,sha256=upUtDZMzwYFU0kzmkelVgkpFToXkrypcR3h_jvGjz14,8596
 cloe_nessy/integration/writer/delta_writer/exceptions.py,sha256=xPmGiYV0xQXauln5Oh34E5vbm0rVcs6xCh-SJSb2bw0,107
@@ -67,7 +67,7 @@ cloe_nessy/pipeline/actions/read_api.py,sha256=RBv5XeHtjTXuCP09Fqo6JNx6iIhQQI-nu
 cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=oXbqbc6BfR82dSIGclwzWiTN8EVmpFjNIYLKm4qOU50,2754
 cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSiivbhWMglyBtkE,7961
 cloe_nessy/pipeline/actions/read_files.py,sha256=N9bFgtG1tovhp2JayxE5YiN9PiO2lgG2-6h_Y6tD2eU,5220
-cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=3ZDy9qiDYtM1oDQzHPC23hLOvHjhdk5zg1wVHE60m9k,2295
+cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=i8fQceV63eAqx_x0ANisCkXWfMHyhqsfFHVFH5yP2po,3544
 cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
 cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=-CEdcXb7Fz5DQNitGlJ8EVBE_LzxfsInyCIO-D7b4iY,3042
 cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
@@ -94,6 +94,6 @@ cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEv
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.13.3b0.dist-info/METADATA,sha256=D5hiAAFKj9BZ6RA5SR9dJ7E4BFQhuNH7AXIT-ELLpP0,3294
-cloe_nessy-0.3.13.3b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cloe_nessy-0.3.13.3b0.dist-info/RECORD,,
+cloe_nessy-0.3.13.4b0.dist-info/METADATA,sha256=Z9Bby2T96l0nm453ZwbiAq3YDP0uau5GfPORfD9KE7E,3294
+cloe_nessy-0.3.13.4b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cloe_nessy-0.3.13.4b0.dist-info/RECORD,,