cloe-nessy 0.3.14.4b0__py3-none-any.whl → 0.3.14.6b0__py3-none-any.whl

cloe_nessy/pipeline/actions/transform_clean_column_names.py

@@ -57,14 +57,14 @@ class TransformCleanColumnNamesAction(PipelineAction):
 
         for c in context.data.schema:
             old_name = c.name
-            new_name = re.sub(single_underscrore_at_beginning, "__", re.sub("\W", "_", old_name))
+            new_name = re.sub(single_underscrore_at_beginning, "__", re.sub(r"\W", "_", old_name))
             with_columns_renamed[old_name] = new_name
 
             if isinstance(c.dataType, (T.StructType | T.ArrayType | T.MapType)):
                 old_column_schema = c.dataType.json()
                 new_column_schema = re.sub(
                     r'(?<="name":")[^"]+',
-                    lambda m: re.sub("\W", "_", str(m.group())),
+                    lambda m: re.sub(r"\W", "_", str(m.group())),
                     old_column_schema,
                 )
                 if isinstance(c.dataType, T.StructType):

cloe_nessy/session/session_manager.py

@@ -5,9 +5,15 @@ from typing import Any
 
 from pyspark.sql import SparkSession
 
+from ..logging import LoggerMixin
 
-class SessionManager:
-    """SessionManager is a singleton class that manages the SparkSession instance."""
+
+class SessionManager(LoggerMixin):
+    """SessionManager is a singleton class that manages the SparkSession instance.
+
+    Logging can be configured via the nessy settings framework. The LoggerMixin provides
+    console logging capabilities with debug-level environment detection information.
+    """
 
     class Environment(Enum):
         """Enumeration of execution environments for Spark utilities.
@@ -60,11 +66,14 @@ class SessionManager:
         nessy_spark_config = os.getenv("NESSY_SPARK_CONFIG")
         if nessy_spark_config:
             try:
-                config = json.loads(nessy_spark_config)
-                if "remote" in config:
-                    builder = builder.remote(config["remote"])
-                    del config["remote"]
-                # Parse the JSON configuration from the environment variable
+                env_config = json.loads(nessy_spark_config)
+                if "remote" in env_config:
+                    builder = builder.remote(env_config["remote"])
+                    del env_config["remote"]
+                if config is None:
+                    config = env_config
+                else:
+                    config.update(env_config)
             except json.JSONDecodeError as e:
                 raise ValueError(f"Invalid JSON in NESSY_SPARK_CONFIG: {e}") from e
 
@@ -152,58 +161,79 @@ class SessionManager:
             RuntimeError: If the environment cannot be detected due to
             import errors or other exceptions.
         """
+        # Create a temporary instance to access LoggerMixin methods
+        temp_instance = cls()
+        logger = temp_instance.get_console_logger()
+
         if cls._env is not None:
+            logger.debug(f"Environment already detected: {cls._env}")
             return cls._env
 
-        try:
-            from databricks.sdk.dbutils import RemoteDbUtils  # type: ignore
+        logger.debug("Starting environment detection...")
 
-            if isinstance(dbutils, RemoteDbUtils):  # type: ignore [name-defined]
-                cls._env = cls.Environment.DATABRICKS_CONNECT
-                return cls._env
-        except (ImportError, NameError):
-            pass
+        # Debug: Print relevant environment variables
+        databricks_host = os.getenv("DATABRICKS_HOST")
+        nessy_spark_config = os.getenv("NESSY_SPARK_CONFIG")
 
-        try:
-            from notebookutils import mssparkutils  # type: ignore # noqa: F401
+        logger.debug(f"DATABRICKS_HOST = {databricks_host}")
+        logger.debug(f"NESSY_SPARK_CONFIG = {nessy_spark_config}")
 
-            cls._env = cls.Environment.FABRIC_UI
-            return cls._env
-        except ImportError:
-            pass
+        if nessy_spark_config:
+            try:
+                config = json.loads(nessy_spark_config)
+                if "remote" in config:
+                    logger.debug(f"Remote Spark configuration detected: {config['remote']}")
+                    cls._env = cls.Environment.OTHER_REMOTE_SPARK
+                    return cls.Environment.OTHER_REMOTE_SPARK
+                cls._env = cls.Environment.STANDALONE_SPARK
+                return cls.Environment.STANDALONE_SPARK
+            except json.JSONDecodeError as e:
+                logger.error(f"Invalid JSON in NESSY_SPARK_CONFIG: {e}")
+                raise ValueError(f"Invalid JSON in NESSY_SPARK_CONFIG: {e}") from e
 
+        logger.debug("Checking for Databricks UI...")
         try:
             from dbruntime.dbutils import DBUtils  # type: ignore [import-not-found]  # noqa: F401
 
+            logger.debug("✓ Detected DATABRICKS_UI via dbruntime.dbutils")
             cls._env = cls.Environment.DATABRICKS_UI
             return cls._env
         except ImportError:
-            pass
+            logger.debug("dbruntime.dbutils not available")
 
+        logger.debug("Checking for Databricks Connect...")
         try:
-            from pyspark.sql.connect.session import (
-                SparkSession as RemoteSparkSession,  # type: ignore [import-not-found]  # noqa: F401
-            )
+            from databricks.sdk.dbutils import RemoteDbUtils  # type: ignore  # noqa: F401
+
+            logger.debug("✓ Detected DATABRICKS_CONNECT via RemoteDbUtils instance")
+            cls._env = cls.Environment.DATABRICKS_CONNECT
+            return cls.Environment.DATABRICKS_CONNECT
 
-            cls._env = cls.Environment.OTHER_REMOTE_SPARK
-            return cls._env
         except ImportError:
-            pass
+            logger.debug("RemoteDbUtils not available")
 
+        logger.debug("Checking for Fabric UI...")
         try:
-            from pyspark.sql import SparkSession  # noqa: F401
+            from notebookutils import mssparkutils  # type: ignore # noqa: F401
 
-            cls._env = cls.Environment.STANDALONE_SPARK
+            logger.debug("✓ Detected FABRIC_UI via notebookutils")
+            cls._env = cls.Environment.FABRIC_UI
             return cls._env
         except ImportError:
-            pass
+            logger.debug("notebookutils not available")
 
-        raise RuntimeError("Cannot detect environment.")
+        logger.error("No environment could be detected")
+        raise RuntimeError(
+            "Cannot detect environment. This usually means you're not in a recognized Spark environment. "
+            "Ensure you're running in a supported environment (Databricks, Fabric, or with proper Spark "
+            "installation configured via NESSY_SPARK_CONFIG)."
+        )
 
     @classmethod
     def get_spark_builder(cls):
         """Get the SparkSession builder based on the current environment."""
-        cls._detect_env()
+        if cls._env is None:
+            cls._detect_env()
         builders = {
             cls.Environment.DATABRICKS_UI: SparkSession.builder,
             cls.Environment.FABRIC_UI: SparkSession.builder,

{cloe_nessy-0.3.14.4b0.dist-info → cloe_nessy-0.3.14.6b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.14.4b0
+Version: 0.3.14.6b0
 Summary: Your friendly datalake monster.
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
 License: MIT
@@ -31,7 +31,6 @@ Requires-Dist: matplotlib<4.0.0,>=3.9.2
 Requires-Dist: types-networkx<4.0.0.0,>=3.2.1.20240820
 Requires-Dist: fsspec<2025.7.1,>=2025.7.0
 Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.8
-Requires-Dist: delta-spark>=3.3.2
 
 # cloe-nessy
 

{cloe_nessy-0.3.14.4b0.dist-info → cloe_nessy-0.3.14.6b0.dist-info}/RECORD

@@ -69,7 +69,7 @@ cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSii
 cloe_nessy/pipeline/actions/read_files.py,sha256=N9bFgtG1tovhp2JayxE5YiN9PiO2lgG2-6h_Y6tD2eU,5220
 cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=3ZDy9qiDYtM1oDQzHPC23hLOvHjhdk5zg1wVHE60m9k,2295
 cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
-cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=-CEdcXb7Fz5DQNitGlJ8EVBE_LzxfsInyCIO-D7b4iY,3042
+cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=VxvWqENW63c50L96JA1V_ioe4By6gGzx_iY86njOXEM,3044
 cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
 cloe_nessy/pipeline/actions/transform_decode.py,sha256=JajMwHREtxa8u_1Q3RZDBVMjncoSel-WzQFVTO0MREg,4455
 cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=E0ypz9qkHMSatNfnHekP-E6svQVL149M4PV02M03drg,5099
@@ -89,12 +89,12 @@ cloe_nessy/pipeline/actions/write_delta_append.py,sha256=fuL29SK9G5K14ycckU3iPex
 cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=Hir7QZZZJ9hmQZXiJ9iz6u06OCmcHFpyKFVB_I1saSM,5043
 cloe_nessy/pipeline/actions/write_file.py,sha256=H8LRst045yij-8XJ5pRB9m5d1lZpZjFa0WSVdSFesPo,2984
 cloe_nessy/session/__init__.py,sha256=t7_YjUhJYW3km_FrucaUdbIl1boQtwkyhw_8yE10qzc,74
-cloe_nessy/session/session_manager.py,sha256=6XGssLpLO-DbWnpvyF_6cPD5Of9VcyB_sBNnV17clW8,7856
+cloe_nessy/session/session_manager.py,sha256=sgsN_U6c_IjYF1wSjd8Opj9FV1gjAvaqVy4ljFjk8AQ,9710
 cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.14.4b0.dist-info/METADATA,sha256=s1pZpHkrRMTLezVfS3VlUABGi8w4KM1SEEZ2RJpZqMY,3328
-cloe_nessy-0.3.14.4b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cloe_nessy-0.3.14.4b0.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
-cloe_nessy-0.3.14.4b0.dist-info/RECORD,,
+cloe_nessy-0.3.14.6b0.dist-info/METADATA,sha256=zz5Bu1hIh0bRbgHua_OOAWCDpnw3R0g_zBkubifTmJo,3294
+cloe_nessy-0.3.14.6b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cloe_nessy-0.3.14.6b0.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.3.14.6b0.dist-info/RECORD,,