cloe-nessy 0.3.14.5b0__py3-none-any.whl → 0.3.14.6__py3-none-any.whl

cloe_nessy/pipeline/actions/transform_clean_column_names.py

@@ -57,14 +57,14 @@ class TransformCleanColumnNamesAction(PipelineAction):
 
         for c in context.data.schema:
             old_name = c.name
-            new_name = re.sub(single_underscrore_at_beginning, "__", re.sub("\W", "_", old_name))
+            new_name = re.sub(single_underscrore_at_beginning, "__", re.sub(r"\W", "_", old_name))
             with_columns_renamed[old_name] = new_name
 
             if isinstance(c.dataType, (T.StructType | T.ArrayType | T.MapType)):
                 old_column_schema = c.dataType.json()
                 new_column_schema = re.sub(
                     r'(?<="name":")[^"]+',
-                    lambda m: re.sub("\W", "_", str(m.group())),
+                    lambda m: re.sub(r"\W", "_", str(m.group())),
                     old_column_schema,
                 )
                 if isinstance(c.dataType, T.StructType):

cloe_nessy/session/session_manager.py

@@ -5,9 +5,15 @@ from typing import Any
 
 from pyspark.sql import SparkSession
 
+from ..logging import LoggerMixin
 
-class SessionManager:
-    """SessionManager is a singleton class that manages the SparkSession instance."""
+
+class SessionManager(LoggerMixin):
+    """SessionManager is a singleton class that manages the SparkSession instance.
+
+    Logging can be configured via the nessy settings framework. The LoggerMixin provides
+    console logging capabilities with debug-level environment detection information.
+    """
 
     class Environment(Enum):
         """Enumeration of execution environments for Spark utilities.
@@ -60,11 +66,14 @@ class SessionManager:
         nessy_spark_config = os.getenv("NESSY_SPARK_CONFIG")
         if nessy_spark_config:
             try:
-                config = json.loads(nessy_spark_config)
-                if "remote" in config:
-                    builder = builder.remote(config["remote"])
-                    del config["remote"]
-                # Parse the JSON configuration from the environment variable
+                env_config = json.loads(nessy_spark_config)
+                if "remote" in env_config:
+                    builder = builder.remote(env_config["remote"])
+                    del env_config["remote"]
+                if config is None:
+                    config = env_config
+                else:
+                    config.update(env_config)
             except json.JSONDecodeError as e:
                 raise ValueError(f"Invalid JSON in NESSY_SPARK_CONFIG: {e}") from e
 
@@ -152,62 +161,68 @@ class SessionManager:
             RuntimeError: If the environment cannot be detected due to
             import errors or other exceptions.
         """
+        # Create a temporary instance to access LoggerMixin methods
+        temp_instance = cls()
+        logger = temp_instance.get_console_logger()
+
         if cls._env is not None:
-            print(f"DEBUG: Environment already detected: {cls._env}")
+            logger.debug(f"Environment already detected: {cls._env}")
             return cls._env
 
-        print("DEBUG: Starting environment detection...")
+        logger.debug("Starting environment detection...")
 
         # Debug: Print relevant environment variables
         databricks_host = os.getenv("DATABRICKS_HOST")
         nessy_spark_config = os.getenv("NESSY_SPARK_CONFIG")
 
-        print(f"DEBUG: DATABRICKS_HOST = {databricks_host}")
-        print(f"DEBUG: NESSY_SPARK_CONFIG = {nessy_spark_config}")
+        logger.debug(f"DATABRICKS_HOST = {databricks_host}")
+        logger.debug(f"NESSY_SPARK_CONFIG = {nessy_spark_config}")
 
         if nessy_spark_config:
             try:
                 config = json.loads(nessy_spark_config)
                 if "remote" in config:
-                    print(f"DEBUG: Remote Spark configuration detected: {config['remote']}")
+                    logger.debug(f"Remote Spark configuration detected: {config['remote']}")
+                    cls._env = cls.Environment.OTHER_REMOTE_SPARK
                     return cls.Environment.OTHER_REMOTE_SPARK
+                cls._env = cls.Environment.STANDALONE_SPARK
                 return cls.Environment.STANDALONE_SPARK
             except json.JSONDecodeError as e:
-                print(f"DEBUG: Invalid JSON in NESSY_SPARK_CONFIG: {e}")
+                logger.error(f"Invalid JSON in NESSY_SPARK_CONFIG: {e}")
                 raise ValueError(f"Invalid JSON in NESSY_SPARK_CONFIG: {e}") from e
 
-        print("DEBUG: Checking for Databricks Connect...")
+        logger.debug("Checking for Databricks UI...")
         try:
-            from databricks.sdk.dbutils import RemoteDbUtils  # type: ignore  # noqa: F401
-
-            print("DEBUG: ✓ Detected DATABRICKS_CONNECT via RemoteDbUtils instance")
-            cls._env = cls.Environment.DATABRICKS_CONNECT
-            return cls.Environment.DATABRICKS_CONNECT
+            from dbruntime.dbutils import DBUtils  # type: ignore [import-not-found]  # noqa: F401
 
+            logger.debug("✓ Detected DATABRICKS_UI via dbruntime.dbutils")
+            cls._env = cls.Environment.DATABRICKS_UI
+            return cls._env
         except ImportError:
-            print("DEBUG: RemoteDbUtils not available")
+            logger.debug("dbruntime.dbutils not available")
 
-        print("DEBUG: Checking for Databricks UI...")
+        logger.debug("Checking for Databricks Connect...")
         try:
-            from dbruntime.dbutils import DBUtils  # type: ignore [import-not-found]  # noqa: F401
+            from databricks.sdk.dbutils import RemoteDbUtils  # type: ignore  # noqa: F401
+
+            logger.debug("✓ Detected DATABRICKS_CONNECT via RemoteDbUtils instance")
+            cls._env = cls.Environment.DATABRICKS_CONNECT
+            return cls.Environment.DATABRICKS_CONNECT
 
-            print("DEBUG: ✓ Detected DATABRICKS_UI via dbruntime.dbutils")
-            cls._env = cls.Environment.DATABRICKS_UI
-            return cls._env
         except ImportError:
-            print("DEBUG: dbruntime.dbutils not available")
+            logger.debug("RemoteDbUtils not available")
 
-        print("DEBUG: Checking for Fabric UI...")
+        logger.debug("Checking for Fabric UI...")
         try:
             from notebookutils import mssparkutils  # type: ignore # noqa: F401
 
-            print("DEBUG: ✓ Detected FABRIC_UI via notebookutils")
+            logger.debug("✓ Detected FABRIC_UI via notebookutils")
             cls._env = cls.Environment.FABRIC_UI
             return cls._env
         except ImportError:
-            print("DEBUG: notebookutils not available")
+            logger.debug("notebookutils not available")
 
-        print("DEBUG: No environment could be detected")
+        logger.error("No environment could be detected")
         raise RuntimeError(
             "Cannot detect environment. This usually means you're not in a recognized Spark environment. "
             "Ensure you're running in a supported environment (Databricks, Fabric, or with proper Spark "

{cloe_nessy-0.3.14.5b0.dist-info → cloe_nessy-0.3.14.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.14.5b0
+Version: 0.3.14.6
 Summary: Your friendly datalake monster.
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
 License: MIT

{cloe_nessy-0.3.14.5b0.dist-info → cloe_nessy-0.3.14.6.dist-info}/RECORD

@@ -69,7 +69,7 @@ cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSii
 cloe_nessy/pipeline/actions/read_files.py,sha256=N9bFgtG1tovhp2JayxE5YiN9PiO2lgG2-6h_Y6tD2eU,5220
 cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=3ZDy9qiDYtM1oDQzHPC23hLOvHjhdk5zg1wVHE60m9k,2295
 cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
-cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=-CEdcXb7Fz5DQNitGlJ8EVBE_LzxfsInyCIO-D7b4iY,3042
+cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=VxvWqENW63c50L96JA1V_ioe4By6gGzx_iY86njOXEM,3044
 cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
 cloe_nessy/pipeline/actions/transform_decode.py,sha256=JajMwHREtxa8u_1Q3RZDBVMjncoSel-WzQFVTO0MREg,4455
 cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=E0ypz9qkHMSatNfnHekP-E6svQVL149M4PV02M03drg,5099
@@ -89,12 +89,12 @@ cloe_nessy/pipeline/actions/write_delta_append.py,sha256=fuL29SK9G5K14ycckU3iPex
 cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=Hir7QZZZJ9hmQZXiJ9iz6u06OCmcHFpyKFVB_I1saSM,5043
 cloe_nessy/pipeline/actions/write_file.py,sha256=H8LRst045yij-8XJ5pRB9m5d1lZpZjFa0WSVdSFesPo,2984
 cloe_nessy/session/__init__.py,sha256=t7_YjUhJYW3km_FrucaUdbIl1boQtwkyhw_8yE10qzc,74
-cloe_nessy/session/session_manager.py,sha256=eyiSiQDpLg6GUwLUiZ8bZbrwh1jN_vYeLiuWFEl7RJo,9123
+cloe_nessy/session/session_manager.py,sha256=sgsN_U6c_IjYF1wSjd8Opj9FV1gjAvaqVy4ljFjk8AQ,9710
 cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.14.5b0.dist-info/METADATA,sha256=nEbjVCZ8v13CJ9gE-KM_U-Uj1hdw5eSfdMPzU9jW3Ls,3294
-cloe_nessy-0.3.14.5b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cloe_nessy-0.3.14.5b0.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
-cloe_nessy-0.3.14.5b0.dist-info/RECORD,,
+cloe_nessy-0.3.14.6.dist-info/METADATA,sha256=7Oe0EDtFCqMLA_y9D0oMqJ-93yy1azqYtKbr-rec9So,3292
+cloe_nessy-0.3.14.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cloe_nessy-0.3.14.6.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.3.14.6.dist-info/RECORD,,