cloe-nessy 0.3.14.3b0__py3-none-any.whl → 0.3.14.5b0__py3-none-any.whl

cloe_nessy/session/session_manager.py

@@ -153,57 +153,72 @@ class SessionManager:
             import errors or other exceptions.
         """
         if cls._env is not None:
+            print(f"DEBUG: Environment already detected: {cls._env}")
             return cls._env
 
-        try:
-            from databricks.sdk.dbutils import RemoteDbUtils  # type: ignore
+        print("DEBUG: Starting environment detection...")
 
-            if isinstance(dbutils, RemoteDbUtils):  # type: ignore [name-defined]
-                cls._env = cls.Environment.DATABRICKS_CONNECT
-                return cls._env
-        except (ImportError, NameError):
-            pass
+        # Debug: Print relevant environment variables
+        databricks_host = os.getenv("DATABRICKS_HOST")
+        nessy_spark_config = os.getenv("NESSY_SPARK_CONFIG")
 
-        try:
-            from notebookutils import mssparkutils  # type: ignore # noqa: F401
+        print(f"DEBUG: DATABRICKS_HOST = {databricks_host}")
+        print(f"DEBUG: NESSY_SPARK_CONFIG = {nessy_spark_config}")
 
-            cls._env = cls.Environment.FABRIC_UI
-            return cls._env
-        except ImportError:
-            pass
+        if nessy_spark_config:
+            try:
+                config = json.loads(nessy_spark_config)
+                if "remote" in config:
+                    print(f"DEBUG: Remote Spark configuration detected: {config['remote']}")
+                    return cls.Environment.OTHER_REMOTE_SPARK
+                return cls.Environment.STANDALONE_SPARK
+            except json.JSONDecodeError as e:
+                print(f"DEBUG: Invalid JSON in NESSY_SPARK_CONFIG: {e}")
+                raise ValueError(f"Invalid JSON in NESSY_SPARK_CONFIG: {e}") from e
 
+        print("DEBUG: Checking for Databricks Connect...")
         try:
-            from dbruntime.dbutils import DBUtils  # type: ignore [import-not-found]  # noqa: F401
+            from databricks.sdk.dbutils import RemoteDbUtils  # type: ignore  # noqa: F401
+
+            print("DEBUG: ✓ Detected DATABRICKS_CONNECT via RemoteDbUtils instance")
+            cls._env = cls.Environment.DATABRICKS_CONNECT
+            return cls.Environment.DATABRICKS_CONNECT
 
-            cls._env = cls.Environment.DATABRICKS_UI
-            return cls._env
         except ImportError:
-            pass
+            print("DEBUG: RemoteDbUtils not available")
 
+        print("DEBUG: Checking for Databricks UI...")
         try:
-            from pyspark.sql.connect.session import (
-                SparkSession as RemoteSparkSession,  # type: ignore [import-not-found]  # noqa: F401
-            )
+            from dbruntime.dbutils import DBUtils  # type: ignore [import-not-found]  # noqa: F401
 
-            cls._env = cls.Environment.OTHER_REMOTE_SPARK
+            print("DEBUG: ✓ Detected DATABRICKS_UI via dbruntime.dbutils")
+            cls._env = cls.Environment.DATABRICKS_UI
             return cls._env
         except ImportError:
-            pass
+            print("DEBUG: dbruntime.dbutils not available")
 
+        print("DEBUG: Checking for Fabric UI...")
         try:
-            from pyspark.sql import SparkSession  # noqa: F401
+            from notebookutils import mssparkutils  # type: ignore # noqa: F401
 
-            cls._env = cls.Environment.STANDALONE_SPARK
+            print("DEBUG: ✓ Detected FABRIC_UI via notebookutils")
+            cls._env = cls.Environment.FABRIC_UI
             return cls._env
         except ImportError:
-            pass
+            print("DEBUG: notebookutils not available")
 
-        raise RuntimeError("Cannot detect environment.")
+        print("DEBUG: No environment could be detected")
+        raise RuntimeError(
+            "Cannot detect environment. This usually means you're not in a recognized Spark environment. "
+            "Ensure you're running in a supported environment (Databricks, Fabric, or with proper Spark "
+            "installation configured via NESSY_SPARK_CONFIG)."
+        )
 
     @classmethod
     def get_spark_builder(cls):
         """Get the SparkSession builder based on the current environment."""
-        cls._detect_env()
+        if cls._env is None:
+            cls._detect_env()
         builders = {
             cls.Environment.DATABRICKS_UI: SparkSession.builder,
             cls.Environment.FABRIC_UI: SparkSession.builder,
@@ -216,7 +231,7 @@ class SessionManager:
             raise ValueError(f"Unsupported environment: {cls._env}")
 
         match cls._env:
-            case cls.Environment.DATABRICKS_CONNECT | cls.Environment.OTHER_REMOTE_SPARK:
+            case cls.Environment.DATABRICKS_CONNECT:
                 return builder()
             case _:
                 return builder

{cloe_nessy-0.3.14.3b0.dist-info → cloe_nessy-0.3.14.5b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.14.3b0
+Version: 0.3.14.5b0
 Summary: Your friendly datalake monster.
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
 License: MIT
@@ -31,7 +31,6 @@ Requires-Dist: matplotlib<4.0.0,>=3.9.2
 Requires-Dist: types-networkx<4.0.0.0,>=3.2.1.20240820
 Requires-Dist: fsspec<2025.7.1,>=2025.7.0
 Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.8
-Requires-Dist: delta-spark>=3.3.2
 
 # cloe-nessy
 

{cloe_nessy-0.3.14.3b0.dist-info → cloe_nessy-0.3.14.5b0.dist-info}/RECORD

@@ -89,12 +89,12 @@ cloe_nessy/pipeline/actions/write_delta_append.py,sha256=fuL29SK9G5K14ycckU3iPex
 cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=Hir7QZZZJ9hmQZXiJ9iz6u06OCmcHFpyKFVB_I1saSM,5043
 cloe_nessy/pipeline/actions/write_file.py,sha256=H8LRst045yij-8XJ5pRB9m5d1lZpZjFa0WSVdSFesPo,2984
 cloe_nessy/session/__init__.py,sha256=t7_YjUhJYW3km_FrucaUdbIl1boQtwkyhw_8yE10qzc,74
-cloe_nessy/session/session_manager.py,sha256=UuTrDO2so_qy89ZtW_QshtxZz5zFSvvuKtBtV0caEa4,7893
+cloe_nessy/session/session_manager.py,sha256=eyiSiQDpLg6GUwLUiZ8bZbrwh1jN_vYeLiuWFEl7RJo,9123
 cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.14.3b0.dist-info/METADATA,sha256=wmr_zFd8SWr8EdBuDzXvfrPh_L9IWBtq0QvBFSSvy6Y,3328
-cloe_nessy-0.3.14.3b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cloe_nessy-0.3.14.3b0.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
-cloe_nessy-0.3.14.3b0.dist-info/RECORD,,
+cloe_nessy-0.3.14.5b0.dist-info/METADATA,sha256=nEbjVCZ8v13CJ9gE-KM_U-Uj1hdw5eSfdMPzU9jW3Ls,3294
+cloe_nessy-0.3.14.5b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cloe_nessy-0.3.14.5b0.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.3.14.5b0.dist-info/RECORD,,