cloe-nessy 0.3.5__py3-none-any.whl → 0.3.8__py3-none-any.whl

cloe_nessy/session/session_manager.py

@@ -1,3 +1,5 @@
+import json
+import os
 from enum import Enum
 from typing import Any
 
@@ -16,12 +18,14 @@ class SessionManager:
             - FABRIC_UI: Represents the Fabric user interface.
             - DATABRICKS_CONNECT: Represents the Databricks Connect environment.
             - OTHER_REMOTE_SPARK: Represents other remote Spark environments, such as used in tests.
+            - STANDALONE_SPARK: Represents a standalone Spark cluster environment.
         """
 
         DATABRICKS_UI = "databricks_ui"
         FABRIC_UI = "fabric_ui"
         DATABRICKS_CONNECT = "databricks_connect"
         OTHER_REMOTE_SPARK = "other_remote_spark"
+        STANDALONE_SPARK = "standalone_spark"
 
     _spark: SparkSession | None = None
     _utils = None
@@ -52,6 +56,15 @@ class SessionManager:
 
         builder = cls.get_spark_builder()
 
+        # Check if NESSY_SPARK_CONFIG environment variable is set and load it as config
+        nessy_spark_config = os.getenv("NESSY_SPARK_CONFIG")
+        if nessy_spark_config:
+            try:
+                # Parse the JSON configuration from the environment variable
+                config = json.loads(nessy_spark_config)
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSON in NESSY_SPARK_CONFIG: {e}") from e
+
         if config:
             for key, value in config.items():
                 builder.config(key, value)  # type: ignore
@@ -84,6 +97,7 @@ class SessionManager:
             cls.Environment.DATABRICKS_UI: cls._get_dbutils,
             cls.Environment.DATABRICKS_CONNECT: cls._get_dbutils,
             cls.Environment.OTHER_REMOTE_SPARK: cls._get_dbutils,
+            cls.Environment.STANDALONE_SPARK: cls._get_localsparkutils,
             cls.Environment.FABRIC_UI: cls._get_mssparkutils,
         }
 
@@ -112,6 +126,10 @@ class SessionManager:
 
         cls._utils = mssparkutils
 
+    @classmethod
+    def _get_localsparkutils(cls):
+        return None
+
     @classmethod
     def _detect_env(cls) -> Environment | None:
         """Detects the current execution environment for Spark.
@@ -169,6 +187,14 @@ class SessionManager:
         except ImportError:
             pass
 
+        try:
+            from pyspark.sql import SparkSession  # noqa: F401
+
+            cls._env = cls.Environment.STANDALONE_SPARK
+            return cls._env
+        except ImportError:
+            pass
+
         raise RuntimeError("Cannot detect environment.")
 
     @classmethod
@@ -180,6 +206,7 @@ class SessionManager:
             cls.Environment.FABRIC_UI: SparkSession.builder,
             cls.Environment.DATABRICKS_CONNECT: cls._get_databricks_connect_builder,
             cls.Environment.OTHER_REMOTE_SPARK: cls._get_databricks_connect_builder,
+            cls.Environment.STANDALONE_SPARK: SparkSession.builder,
         }
         builder = builders.get(cls._env)
         if builder is None:

cloe_nessy-0.3.8.dist-info/METADATA

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: cloe-nessy
+Version: 0.3.8
+Summary: Your friendly datalake monster.
+Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
+License: MIT
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Requires-Python: <3.12,>=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic<3.0.0,>=2.7.2
+Requires-Dist: pyyaml<7.0.0,>=6.0.1
+Requires-Dist: types-pyyaml<7.0.0.0,>=6.0.12.20240311
+Requires-Dist: jinja2<4.0.0,>=3.1.4
+Requires-Dist: pydantic-settings<3.0.0,>=2.4.0
+Requires-Dist: openpyxl<4.0.0,>=3.1.5
+Requires-Dist: requests<3.0.0,>=2.32.3
+Requires-Dist: types-requests<3.0.0.0,>=2.32.0.20240712
+Requires-Dist: pandas-stubs<3.0.0.0,>=2.2.2.240807
+Requires-Dist: azure-identity<2.0.0,>=1.19.0
+Requires-Dist: httpx<1.0.0,>=0.27.2
+Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
+Requires-Dist: networkx<4.0,>=3.3
+Requires-Dist: matplotlib<4.0.0,>=3.9.2
+Requires-Dist: types-networkx<4.0.0.0,>=3.2.1.20240820
+Requires-Dist: fsspec<2025.0.0,>=2024.9.0
+Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.7
+
+# cloe-nessy
+
+[![Copier](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/copier-org/copier/master/img/badge/badge-grayscale-inverted-border-orange.json)](https://github.com/copier-org/copier)
+[![python](https://img.shields.io/badge/Python-3.12-3776AB.svg?style=flat&logo=python&logoColor=white)](https://www.python.org)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v0.json)](https://github.com/charliermarsh/ruff)
+[![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit)
+
+Owner: initions
+
+Your friendly datalake monster.

{cloe_nessy-0.3.5.dist-info → cloe_nessy-0.3.8.dist-info}/RECORD

@@ -28,26 +28,31 @@ cloe_nessy/integration/writer/__init__.py,sha256=NIh0t1RYlG3J1Y5_CvnR36N9tISmcEl
 cloe_nessy/integration/writer/catalog_writer.py,sha256=Gb-hMdADgO_uUJ7mZPHBYyNme2qXsdFFnzwo7GcShHM,2192
 cloe_nessy/logging/__init__.py,sha256=ySVCVbdyR3Dno_tl2ZfiER_7EVaDoQMHVkNyfdMZumY,65
 cloe_nessy/logging/logger_mixin.py,sha256=9iy7BF6drYme-f7Rrt_imbVBRgVqQ89xjcP1X5aMtfY,7467
-cloe_nessy/models/__init__.py,sha256=_JPN_R5-QDfjYzvrvZDdeOezl0C-JTG-Rk4S1VE5vJM,242
+cloe_nessy/models/__init__.py,sha256=-FmWEJ1Oq1njSopjc0R7GmT64mLSmALkm8PkHNzy9Y8,327
+cloe_nessy/models/catalog.py,sha256=ayC1sMp4cNLAZtu0ICVV3Us6-o4hn8U9tpzzvxC9RAs,177
 cloe_nessy/models/column.py,sha256=53fBwRnino72XKACsHZpN9QfCBqqSXyKLHZlM0huumg,1988
 cloe_nessy/models/constraint.py,sha256=hsFlhn4n928z81O3dl3v5bMetewPWzMjkJK3_4kASSM,178
 cloe_nessy/models/foreign_key.py,sha256=DwRVHs9sShqqPV-NL7ow_3AmPPWX0Od26yZn_I565pU,1001
-cloe_nessy/models/schema.py,sha256=8bc1fakLUWZzkVZ_Zn5iWMUvfDNxnuoToNE4kmqtBJo,2764
-cloe_nessy/models/table.py,sha256=lshPBA3D6vA1samtC7WmlfZZWrMUrOLna89rs8lhGCI,10472
+cloe_nessy/models/schema.py,sha256=DHlJJ4JA8392vVrFatH22GoPOCJN-eqkmHVihIHpPCg,3382
+cloe_nessy/models/table.py,sha256=W9gqMTCQBRo3Z_rLY5cJ8WgPEUEslNLbaRrODuTy15Y,11744
 cloe_nessy/models/types.py,sha256=XRbuJGdTNa6aXyE3IAzs_J9gVjbfkzMDLfGl-k6jI_4,223
+cloe_nessy/models/volume.py,sha256=7_lH4X-Au8jYNRVFQ5F2x-fRy2J8Z5-cLTn3Z3mpLQs,2197
+cloe_nessy/models/adapter/__init__.py,sha256=m36W_mqwB3dCYnCIt0fLOSHS4E1VU8FRGoaum4Gf95o,90
+cloe_nessy/models/adapter/unity_catalog_adapter.py,sha256=T5v8gN57vRj9OLrfj8N4TEsH_Z_5N38wTqu_7rYgXb0,12572
 cloe_nessy/models/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/models/mixins/read_instance_mixin.py,sha256=j5Y4aNWOh1jlskEaxNooZFJgPyxRmik00gAVLJnAaRs,4507
 cloe_nessy/models/mixins/template_loader_mixin.py,sha256=5MXhEGBFlq3dwZvINEyBowSlipNnVun2H_TmhI_fsS4,549
-cloe_nessy/object_manager/__init__.py,sha256=tt_sBt8eC-jCP8FShqRPKJvGNVIPeb-htA7NoUivTjY,68
-cloe_nessy/object_manager/table_manager.py,sha256=K6OGCNNDt1ceLA0MkwtyW6AR5tYIW3tfqF3ZcvHlcUw,2717
+cloe_nessy/object_manager/__init__.py,sha256=3sle0vNpPwBOkycxA3XVS9m4XZf5LD3Qd4NGxdqcHno,186
+cloe_nessy/object_manager/table_manager.py,sha256=B_00UXaEQQHTAsyZGivhW3TQQpejC3obljAi_IwyFNA,9993
+cloe_nessy/object_manager/volume_manager.py,sha256=6epd3KXzcNH04EvaKubAfLsaUm9qBMeT3KNvMK04gGs,2727
 cloe_nessy/pipeline/__init__.py,sha256=sespmJ5JsgyiFyZiedTiL2kg--zGIX7cjTYsD5vemEg,325
-cloe_nessy/pipeline/pipeline.py,sha256=oQ1PwYkOSGHOfgbmImy7IbB5Ma-NKHN_CMXq1FepTc4,9206
+cloe_nessy/pipeline/pipeline.py,sha256=-1tJVs9rZf8CcwieH4IP7mqJZ6mL7bQUZ56TNKt8eO8,11154
 cloe_nessy/pipeline/pipeline_action.py,sha256=S7IVFdmG12fRBzHuE_DiWn7qlMtApz6IloVd2Fj31Sg,1944
 cloe_nessy/pipeline/pipeline_config.py,sha256=BN3ZSbr6bC-X9edoh-n5vRfPHFMbgtAU7mQ3dBrcWO8,3131
 cloe_nessy/pipeline/pipeline_context.py,sha256=csElDc6BsynDUtRXgQOSCH7ONc_b-ag0YEg0zlQTz58,1874
 cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=c_nAsgw81QYBM9AFiTxGgqRhNXABkDKplbeoCJPtbpE,6434
 cloe_nessy/pipeline/pipeline_step.py,sha256=UlnmpS6gm_dZ7m9dD1mZvye7mvUF_DA7HjOZo0oGYDU,1977
-cloe_nessy/pipeline/actions/__init__.py,sha256=LwKctXy4Jun52BnCVGvWa8nnKVjTSov4GT58j6Zy8zg,2273
+cloe_nessy/pipeline/actions/__init__.py,sha256=dvIaWVR-L3IQkxMI-TIazn2udeFGsKmhUxw9E0VTz0g,2370
 cloe_nessy/pipeline/actions/read_api.py,sha256=RBv5XeHtjTXuCP09Fqo6JNx6iIhQQI-nuAHCuSaGs2s,7778
 cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=-k2wezkv8bE_xwoW7WM1ORhrCXQagKTUuXkhI2ZEROs,2783
 cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSiivbhWMglyBtkE,7961
@@ -62,6 +67,7 @@ cloe_nessy/pipeline/actions/transform_distinct.py,sha256=c7aBxANyqT4aKhm0cSELDtD
 cloe_nessy/pipeline/actions/transform_filter.py,sha256=Nz_ggRfKIcNzYFfFOsgq1QeatjdEis0up4I7cOWBdyo,1446
 cloe_nessy/pipeline/actions/transform_generic_sql.py,sha256=_naWfmPdYAUKjPNeHu5qJAohOL7DHCSYz_kwoeRv3OI,2741
 cloe_nessy/pipeline/actions/transform_group_aggregate.py,sha256=KUHeeP-RIDi34dpbsPEJkzea5zFJA6MuyjNpOsFud9o,4045
+cloe_nessy/pipeline/actions/transform_hash_columns.py,sha256=heRjBA-Gfu-nmNHOjTYlipEpKY8oNPAHAY40vjJk3aI,8383
 cloe_nessy/pipeline/actions/transform_join.py,sha256=e_tvMk8YJTAWcUK_EmOgNt0s31ICZoMX_MKOTWx4lBY,3645
 cloe_nessy/pipeline/actions/transform_json_normalize.py,sha256=petF7pnNq1EKc8MqVdG0weFALAHNILSe_eAu4Z5XxIo,4833
 cloe_nessy/pipeline/actions/transform_rename_columns.py,sha256=4zJcPCONMU4C67qeuzsrX3AORRRHoq_selUI7FJyeg0,1952
@@ -70,12 +76,12 @@ cloe_nessy/pipeline/actions/transform_select_columns.py,sha256=-GhSEsb7iNnZIsYRm
 cloe_nessy/pipeline/actions/transform_union.py,sha256=s81Vge0AbYPc7VkskCYfOQ_LEjqcmfNFyDkytfjcZyo,2720
 cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=j7gRuG3Fedh8JgevIFBbHKock3laJVq4l6Mx3CGU5eo,2676
 cloe_nessy/session/__init__.py,sha256=t7_YjUhJYW3km_FrucaUdbIl1boQtwkyhw_8yE10qzc,74
-cloe_nessy/session/session_manager.py,sha256=PK7awMc6fmot7f9FMmvIUbIzKFgjcy2o2bZS9kjVs10,6733
+cloe_nessy/session/session_manager.py,sha256=f4OeeyGD3becDQGkdDbck3jVH9ulOCBWjW6Jaj_MIrc,7765
 cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.5.dist-info/METADATA,sha256=UUx3aIUgvCLn7j3H4DbCL1k9-47HPKaANiMQsUj66wo,1837
-cloe_nessy-0.3.5.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-cloe_nessy-0.3.5.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
-cloe_nessy-0.3.5.dist-info/RECORD,,
+cloe_nessy-0.3.8.dist-info/METADATA,sha256=fnnrNC7Vo8REfpUO5hdflH0_zaZAF437CyRrUuCM8dI,2374
+cloe_nessy-0.3.8.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
+cloe_nessy-0.3.8.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.3.8.dist-info/RECORD,,

{cloe_nessy-0.3.5.dist-info → cloe_nessy-0.3.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.0.0)
+Generator: setuptools (80.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

cloe_nessy-0.3.5.dist-info/METADATA

@@ -1,26 +0,0 @@
-Metadata-Version: 2.2
-Name: cloe-nessy
-Version: 0.3.5
-Summary: Your friendly datalake monster.
-Home-page: https://initions.com/
-Author: initions
-Author-email: ICSMC_EXT_PYPIORG@accenture.com
-License: MIT
-Requires-Python: <3.12,>=3.11
-Requires-Dist: pydantic<3.0.0,>=2.7.2
-Requires-Dist: pyyaml<7.0.0,>=6.0.1
-Requires-Dist: types-pyyaml<7.0.0.0,>=6.0.12.20240311
-Requires-Dist: jinja2<4.0.0,>=3.1.4
-Requires-Dist: pydantic-settings<3.0.0,>=2.4.0
-Requires-Dist: openpyxl<4.0.0,>=3.1.5
-Requires-Dist: requests<3.0.0,>=2.32.3
-Requires-Dist: types-requests<3.0.0.0,>=2.32.0.20240712
-Requires-Dist: pandas-stubs<3.0.0.0,>=2.2.2.240807
-Requires-Dist: azure-identity<2.0.0,>=1.19.0
-Requires-Dist: httpx<1.0.0,>=0.27.2
-Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
-Requires-Dist: networkx<4.0,>=3.3
-Requires-Dist: matplotlib<4.0.0,>=3.9.2
-Requires-Dist: types-networkx<4.0.0.0,>=3.2.1.20240820
-Requires-Dist: fsspec<2025.0.0,>=2024.9.0
-Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.7