cloe-nessy 0.3.13.4b0__py3-none-any.whl → 0.3.14.1b0__py3-none-any.whl

cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py

@@ -196,11 +196,7 @@ class DeltaMergeWriter(BaseDeltaWriter):
 
         config = DeltaMergeConfig(dataframe_columns=data_frame.columns, **kwargs)
 
-        delta_table = self.table_manager.get_delta_table(
-            table=table,
-            location=storage_path,
-            spark=data_frame.sparkSession,
-        )
+        delta_table = self.table_manager.get_delta_table(location=storage_path, spark=data_frame.sparkSession)
 
         match_conditions = self._build_match_conditions(data_frame, config)
 

cloe_nessy/models/schema.py

@@ -43,7 +43,6 @@ class Schema(ReadInstancesMixin):
             raise FileNotFoundError("Schema file not found.")
 
         schema, schema_errors = super().read_instance_from_file(processed_instance_path)
-        table_errors: list[ValidationErrorType] = []
         if schema:
             schema.storage_path = "" if not schema.storage_path else schema.storage_path
             tables, table_errors = Table.read_instances_from_directory(

cloe_nessy/object_manager/table_manager.py

@@ -186,9 +186,6 @@ class TableManager(LoggerMixin):
     def get_delta_table(self, table: Table | None = None, location: str | None = None, spark=None) -> DeltaTable:
         """Get the DeltaTable object from the Table objects location or a location string.
 
-        For managed tables, uses the table identifier to access the DeltaTable.
-        For external tables or when a location is provided, uses the storage path.
-
         Args:
             table: A Table object representing the Delta table.
             location: A string representing the table location.
@@ -198,34 +195,15 @@ class TableManager(LoggerMixin):
             The DeltaTable object corresponding to the given Table object or location string.
 
         Raises:
-            ValueError: If neither table nor location is provided.
+            ValueError: If neither table nor location is provided, or if both are provided.
         """
-        if table is None and location is None:
-            self._console_logger.error("Invalid parameters: both table and location are None")
-            raise ValueError("Either table or location must be provided.")
-
-        spark_session = spark or self._spark
-
-        if table is not None and location is not None:
-            self._console_logger.info(
-                f"Both table ({table.identifier}) and location ({location}) provided. Using table object as priority."
-            )
+        if (table is None and location is None) or (table is not None and location is not None):
+            raise ValueError("Either table or location must be provided, but not both.")
 
         if table is not None:
-            if table.is_external is False:
-                self._console_logger.info(f"Getting DeltaTable object for managed table: {table.identifier}")
-                return DeltaTable.forName(spark_session, table.identifier)
-
-            table_location = str(table.storage_path)
-            self._console_logger.info(f"Getting DeltaTable object for external table location: {table_location}")
-            return DeltaTable.forPath(spark_session, table_location)
-
-        self._console_logger.info(f"No table object provided, using location: {location}")
-        if location is None:
-            self._console_logger.error("Location is None - this should not happen!")
-            raise ValueError("Location cannot be None when no table object is provided")
+            location = str(table.storage_path)
         self._console_logger.info(f"Getting DeltaTable object for location: {location}")
-        return DeltaTable.forPath(spark_session, str(location))
+        return DeltaTable.forPath(spark or self._spark, str(location))
 
     def table_exists(self, table: Table | None = None, table_identifier: str | None = None) -> bool:
         """Checks if a table exists in the catalog.
@@ -255,10 +233,9 @@ class TableManager(LoggerMixin):
                 raise ValueError("Invalid table identifier format. Expected 'catalog.schema.table'.")
 
         query_result = self._spark.sql(
-            # Using both upper and lower case to ensure compatibility with case changes in Databricks
             f"""
                 SELECT 1 FROM {catalog}.information_schema.tables
-                WHERE table_name in ('{table_name}', '{table_name.lower()}')
+                WHERE table_name = '{table_name}'
                 AND table_schema = '{schema}'
                 LIMIT 1""",
         )

cloe_nessy/pipeline/actions/read_metadata_yaml.py

@@ -1,94 +1,66 @@
-from pathlib import Path
+import pathlib
 from typing import Any
 
-from ...models import Table
+from ...models import Schema
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
 
 
 class ReadMetadataYAMLAction(PipelineAction):
-    """Reads table metadata from a yaml file using the [`Table`][cloe_nessy.models.table] model.
+    """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
 
     Example:
-        === "Managed Table"
-            ```yaml
-            Read Table Metadata:
-                action: READ_METADATA_YAML_ACTION
-                options:
-                    file_path: metadata/schemas/bronze/sales_table.yml
-                    catalog_name: production
-                    schema_name: sales_data
-            ```
-        === "External Table"
-            ```yaml
-            Read Table Metadata:
-                action: READ_METADATA_YAML_ACTION
-                options:
-                    file_path: metadata/schemas/bronze/sales_table.yml
-                    catalog_name: production
-                    schema_name: sales_data
-                    storage_path: abfs://external_storage/sales_data/sales_table
-            ```
+        ```yaml
+        Read Schema Metadata:
+            action: READ_METADATA_YAML_ACTION
+            options:
+                path: excel_file_folder/excel_files_june/
+                file_name: sales_schema.yml
+                table_name: sales
+        ```
     """
 
     name: str = "READ_METADATA_YAML_ACTION"
 
+    @staticmethod
     def run(
-        self,
         context: PipelineContext,
         *,
-        file_path: str | None = None,
-        catalog_name: str | None = None,
-        schema_name: str | None = None,
-        storage_path: str | None = None,
+        path: str | None = None,
+        file_name: str | None = None,
+        table_name: str | None = None,
         **_: Any,
     ) -> PipelineContext:
-        """Reads table metadata from a yaml file using the [`Table`][cloe_nessy.models.table] model.
+        """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
 
         Args:
             context: The context in which this Action is executed.
-            file_path: The path to the file that defines the table.
-            catalog_name: The name of the catalog for the table.
-            schema_name: The name of the schema for the table.
-            storage_path: The storage path for the table, if applicable. If not
-                provided, the table will be considered a managed table.
+            path: The path to the data contract directory.
+            file_name: The name of the file that defines the schema.
+            table_name: The name of the table for which to retrieve metadata.
 
         Raises:
-            ValueError: If any issues occur while reading the table metadata, such as an invalid table,
-                missing file, missing path, or missing catalog/schema names.
+            ValueError: If any issues occur while reading the schema, such as an invalid schema,
+                missing file, or missing path.
 
         Returns:
             The context after the execution of this Action, containing the table metadata.
         """
-        missing_params = []
-        if not file_path:
-            missing_params.append("file_path")
-        if not catalog_name:
-            missing_params.append("catalog_name")
-        if not schema_name:
-            missing_params.append("schema_name")
+        if not path:
+            raise ValueError("No path provided. Please specify path to schema metadata.")
+        if not file_name:
+            raise ValueError("No file_name provided. Please specify file name.")
+        if not table_name:
+            raise ValueError("No table_name provided. Please specify table name.")
 
-        if missing_params:
-            raise ValueError(
-                f"Missing required parameters: {', '.join(missing_params)}. Please specify all required parameters."
-            )
+        path_obj = pathlib.Path(path)
 
-        final_file_path = Path(file_path) if file_path else Path()
-
-        table, errors = Table.read_instance_from_file(
-            final_file_path,
-            catalog_name=catalog_name,
-            schema_name=schema_name,
-        )
+        schema, errors = Schema.read_instance_from_file(path_obj / file_name)
         if errors:
-            raise ValueError(f"Errors while reading table metadata: {errors}")
-        if not table:
-            raise ValueError("No table found in metadata.")
+            raise ValueError(f"Errors while reading schema metadata: {errors}")
+        if not schema:
+            raise ValueError("No schema found in metadata.")
 
-        if not table.storage_path and storage_path:
-            self._console_logger.info(f"Setting storage path for table [ '{table.name}' ] to [ '{storage_path}' ]")
-            table.storage_path = storage_path
-            table.is_external = True
+        table = schema.get_table_by_name(table_name=table_name)
 
-        self._console_logger.info(f"Table [ '{table.name}' ] metadata read successfully from [ '{file_path}' ]")
         return context.from_existing(table_metadata=table)

cloe_nessy/pipeline/actions/write_delta_merge.py

@@ -98,7 +98,6 @@ class WriteDeltaMergeAction(PipelineAction):
 
         delta_merge_writer.write(
             table_identifier=context.table_metadata.identifier,
-            table=context.table_metadata,
             storage_path=str(context.table_metadata.storage_path),
             data_frame=context.data,
             key_columns=key_columns,

cloe_nessy/session/session_manager.py

@@ -60,8 +60,11 @@ class SessionManager:
         nessy_spark_config = os.getenv("NESSY_SPARK_CONFIG")
         if nessy_spark_config:
             try:
-                # Parse the JSON configuration from the environment variable
                 config = json.loads(nessy_spark_config)
+                if "remote" in config:
+                    builder = builder.remote(config["remote"])
+                    del config["remote"]
+                # Parse the JSON configuration from the environment variable
             except json.JSONDecodeError as e:
                 raise ValueError(f"Invalid JSON in NESSY_SPARK_CONFIG: {e}") from e
 

{cloe_nessy-0.3.13.4b0.dist-info → cloe_nessy-0.3.14.1b0.dist-info}/METADATA

@@ -1,36 +1,36 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.13.4b0
+Version: 0.3.14.1b0
 Summary: Your friendly datalake monster.
-Project-URL: homepage, https://initions.com/
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
 License: MIT
+Project-URL: homepage, https://initions.com/
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
-Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
+Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: <3.13,>=3.11
-Requires-Dist: azure-identity<2.0.0,>=1.19.0
-Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.8
-Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
-Requires-Dist: fsspec<2025.6.0,>=2025.5.1
-Requires-Dist: httpx<1.0.0,>=0.27.2
-Requires-Dist: jinja2<4.0.0,>=3.1.4
-Requires-Dist: matplotlib<4.0.0,>=3.9.2
-Requires-Dist: networkx<4.0,>=3.3
-Requires-Dist: openpyxl<4.0.0,>=3.1.5
-Requires-Dist: pandas-stubs<3.0.0.0,>=2.2.2.240807
-Requires-Dist: pydantic-settings<3.0.0,>=2.4.0
+Description-Content-Type: text/markdown
 Requires-Dist: pydantic<3.0.0,>=2.7.2
 Requires-Dist: pyyaml<7.0.0,>=6.0.1
-Requires-Dist: requests<3.0.0,>=2.32.3
-Requires-Dist: types-networkx<4.0.0.0,>=3.2.1.20240820
 Requires-Dist: types-pyyaml<7.0.0.0,>=6.0.12.20240311
+Requires-Dist: jinja2<4.0.0,>=3.1.4
+Requires-Dist: pydantic-settings<3.0.0,>=2.4.0
+Requires-Dist: openpyxl<4.0.0,>=3.1.5
+Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-requests<3.0.0.0,>=2.32.0.20240712
-Description-Content-Type: text/markdown
+Requires-Dist: pandas-stubs<3.0.0.0,>=2.2.2.240807
+Requires-Dist: azure-identity<2.0.0,>=1.19.0
+Requires-Dist: httpx<1.0.0,>=0.27.2
+Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
+Requires-Dist: networkx<4.0,>=3.3
+Requires-Dist: matplotlib<4.0.0,>=3.9.2
+Requires-Dist: types-networkx<4.0.0.0,>=3.2.1.20240820
+Requires-Dist: fsspec<2025.7.1,>=2025.7.0
+Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.8
 
 # cloe-nessy
 

{cloe_nessy-0.3.13.4b0.dist-info → cloe_nessy-0.3.14.1b0.dist-info}/RECORD

@@ -30,7 +30,7 @@ cloe_nessy/integration/writer/file_writer.py,sha256=SUDbN13ZzDhbM8DpOGFgM_Gkg70T
 cloe_nessy/integration/writer/writer.py,sha256=elFPLFrWR-qVE9qnBtzzzhyRALLQcRVuOsPS0rNmRt4,1741
 cloe_nessy/integration/writer/delta_writer/__init__.py,sha256=h2CT6Hllmk0nodlek27uqwniCzVZKMkYcPGyG9K2Z24,164
 cloe_nessy/integration/writer/delta_writer/delta_append_writer.py,sha256=TbpW-j87_H9dcUza34uR6VWslJez406y3_5N1ip0SnM,4740
-cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=zhqPIPfAJTzSLFgBUCwFesUW7CcF1zCPRU-N_8yYjok,10172
+cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=no2GOLqMAJd0fEy2mqMevMj_CvutcJPRmXJC2tD4icA,10112
 cloe_nessy/integration/writer/delta_writer/delta_table_operation_type.py,sha256=kiacqQ2FYQSzakJqZ9-ZHH3os4X7--QuER_2xx9y21k,971
 cloe_nessy/integration/writer/delta_writer/delta_writer_base.py,sha256=upUtDZMzwYFU0kzmkelVgkpFToXkrypcR3h_jvGjz14,8596
 cloe_nessy/integration/writer/delta_writer/exceptions.py,sha256=xPmGiYV0xQXauln5Oh34E5vbm0rVcs6xCh-SJSb2bw0,107
@@ -41,7 +41,7 @@ cloe_nessy/models/catalog.py,sha256=ayC1sMp4cNLAZtu0ICVV3Us6-o4hn8U9tpzzvxC9RAs,
 cloe_nessy/models/column.py,sha256=53fBwRnino72XKACsHZpN9QfCBqqSXyKLHZlM0huumg,1988
 cloe_nessy/models/constraint.py,sha256=hsFlhn4n928z81O3dl3v5bMetewPWzMjkJK3_4kASSM,178
 cloe_nessy/models/foreign_key.py,sha256=DwRVHs9sShqqPV-NL7ow_3AmPPWX0Od26yZn_I565pU,1001
-cloe_nessy/models/schema.py,sha256=cNSrH7K4hLRrkg1E6fW6DUIBMZdR2A5B21POj5iQ4GA,3429
+cloe_nessy/models/schema.py,sha256=yUrjjEhAH5zbCymE67Az_jPnVB8hGO-_UNfqzeZCD_Y,3376
 cloe_nessy/models/table.py,sha256=O9vcJ1XBIb6kA-NAI3SNpB5b7MGDo3p4wMJdonPaBfA,12076
 cloe_nessy/models/types.py,sha256=XRbuJGdTNa6aXyE3IAzs_J9gVjbfkzMDLfGl-k6jI_4,223
 cloe_nessy/models/volume.py,sha256=51BE06FrL1Wv6zblFwJ_HTiR6WQqH7pSmrdH90rqwLg,2444
@@ -53,7 +53,7 @@ cloe_nessy/models/mixins/template_loader_mixin.py,sha256=5MXhEGBFlq3dwZvINEyBowS
 cloe_nessy/models/templates/create_table.sql.j2,sha256=QWbiTXwmGaIlZUAIGL4pAlHkDbP9mq1vGAkdKCPOqm4,1669
 cloe_nessy/models/templates/create_volume.sql.j2,sha256=XIUf1cHcvAxcGTyhzUiv4xpQ1cfDw_ra3_FKmOuLoBs,289
 cloe_nessy/object_manager/__init__.py,sha256=3sle0vNpPwBOkycxA3XVS9m4XZf5LD3Qd4NGxdqcHno,186
-cloe_nessy/object_manager/table_manager.py,sha256=suHx56TYXagaJ2dVkvTP7vwSI4xgTqXNkHYBbYh2pd4,13913
+cloe_nessy/object_manager/table_manager.py,sha256=oYcYiZR0-JyoadcCcDelxfFb-ATeKDIZerYaZc-moiI,12634
 cloe_nessy/object_manager/volume_manager.py,sha256=6epd3KXzcNH04EvaKubAfLsaUm9qBMeT3KNvMK04gGs,2727
 cloe_nessy/pipeline/__init__.py,sha256=sespmJ5JsgyiFyZiedTiL2kg--zGIX7cjTYsD5vemEg,325
 cloe_nessy/pipeline/pipeline.py,sha256=-1tJVs9rZf8CcwieH4IP7mqJZ6mL7bQUZ56TNKt8eO8,11154
@@ -67,7 +67,7 @@ cloe_nessy/pipeline/actions/read_api.py,sha256=RBv5XeHtjTXuCP09Fqo6JNx6iIhQQI-nu
 cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=oXbqbc6BfR82dSIGclwzWiTN8EVmpFjNIYLKm4qOU50,2754
 cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSiivbhWMglyBtkE,7961
 cloe_nessy/pipeline/actions/read_files.py,sha256=N9bFgtG1tovhp2JayxE5YiN9PiO2lgG2-6h_Y6tD2eU,5220
-cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=i8fQceV63eAqx_x0ANisCkXWfMHyhqsfFHVFH5yP2po,3544
+cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=3ZDy9qiDYtM1oDQzHPC23hLOvHjhdk5zg1wVHE60m9k,2295
 cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
 cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=-CEdcXb7Fz5DQNitGlJ8EVBE_LzxfsInyCIO-D7b4iY,3042
 cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
@@ -86,14 +86,15 @@ cloe_nessy/pipeline/actions/transform_select_columns.py,sha256=-GhSEsb7iNnZIsYRm
 cloe_nessy/pipeline/actions/transform_union.py,sha256=s81Vge0AbYPc7VkskCYfOQ_LEjqcmfNFyDkytfjcZyo,2720
 cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=j7gRuG3Fedh8JgevIFBbHKock3laJVq4l6Mx3CGU5eo,2676
 cloe_nessy/pipeline/actions/write_delta_append.py,sha256=fuL29SK9G5K14ycckU3iPexeK0XNXUfQscCwhXHxbKA,2498
-cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=gh3oD0ZGjDq0hw56NiRimK4HHCruDofqqdzFFgYLve8,5085
+cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=Hir7QZZZJ9hmQZXiJ9iz6u06OCmcHFpyKFVB_I1saSM,5043
 cloe_nessy/pipeline/actions/write_file.py,sha256=H8LRst045yij-8XJ5pRB9m5d1lZpZjFa0WSVdSFesPo,2984
 cloe_nessy/session/__init__.py,sha256=t7_YjUhJYW3km_FrucaUdbIl1boQtwkyhw_8yE10qzc,74
-cloe_nessy/session/session_manager.py,sha256=f4OeeyGD3becDQGkdDbck3jVH9ulOCBWjW6Jaj_MIrc,7765
+cloe_nessy/session/session_manager.py,sha256=whWEXenVKan4xy99Y2vShEe4BDcq1viGLUNRW-PyyKo,7908
 cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.13.4b0.dist-info/METADATA,sha256=Z9Bby2T96l0nm453ZwbiAq3YDP0uau5GfPORfD9KE7E,3294
-cloe_nessy-0.3.13.4b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cloe_nessy-0.3.13.4b0.dist-info/RECORD,,
+cloe_nessy-0.3.14.1b0.dist-info/METADATA,sha256=1GJtUcYEA_BhjoT-op3vn-lG5o19s3hFqgFGF9UXKJc,3294
+cloe_nessy-0.3.14.1b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cloe_nessy-0.3.14.1b0.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.3.14.1b0.dist-info/RECORD,,

{cloe_nessy-0.3.13.4b0.dist-info → cloe_nessy-0.3.14.1b0.dist-info}/WHEEL

@@ -1,4 +1,5 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
+

cloe_nessy-0.3.14.1b0.dist-info/top_level.txt

@@ -0,0 +1 @@
+cloe_nessy