satisfactoscript 0.5.5__tar.gz → 0.5.7__tar.gz

{satisfactoscript-0.5.5 → satisfactoscript-0.5.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: satisfactoscript
-Version: 0.5.5
+Version: 0.5.7
 Summary: An Enterprise-Ready, Declarative Data Engineering Framework for Databricks Lakehouse.
 Author: julhouba
 Classifier: Programming Language :: Python :: 3

{satisfactoscript-0.5.5 → satisfactoscript-0.5.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "satisfactoscript"
-version = "0.5.5"
+version = "0.5.7"
 description = "An Enterprise-Ready, Declarative Data Engineering Framework for Databricks Lakehouse."
 readme = "README.md"
 authors = [

{satisfactoscript-0.5.5 → satisfactoscript-0.5.7}/src/satisfactoscript/core/core.py

@@ -188,7 +188,7 @@ class SatisfactoEngine:
                     # Use the modern approach for Databricks Connect v2
                     from databricks.connect import DatabricksSession
                     from databricks.sdk.core import Config
-                    
+
                     # We explicitly inject the values from the environment variables
                     # into the Config object so Databricks Connect doesn't rely solely
                     # on the implicit ~\.databrickscfg profile.
@@ -197,7 +197,7 @@ class SatisfactoEngine:
                         token=os.getenv("DATABRICKS_TOKEN"),
                         cluster_id=os.getenv("DATABRICKS_CLUSTER_ID")
                     )
-                        
+
                     self.spark = DatabricksSession.builder.sdkConfig(conf).getOrCreate()
                     print("   [Init] 🚀 Remote DatabricksSession (Databricks Connect) initialized.")
                 except Exception as e:
@@ -294,11 +294,43 @@ class SatisfactoEngine:
         # Auto-detect environment based on priority list
         self._auto_detect_environment()
 
+    def _check_catalog_access(self, catalog):
+        """
+        Verifies access to a catalog using two strategies in order:
+        1. Spark SQL SHOW SCHEMAS (native on Databricks platform/cluster).
+        2. Databricks SDK REST API (works locally via Databricks Connect when
+           gRPC DDL calls fail with 'Missing UserContext').
+
+        Returns:
+            bool: True if the catalog is accessible, False otherwise.
+        """
+        # Strategy 1: Spark SQL (works natively on Databricks)
+        try:
+            self.spark.sql(f"SHOW SCHEMAS IN `{catalog}`").limit(1).collect()
+            return True
+        except Exception:
+            pass
+
+        # Strategy 2: Databricks SDK REST API (works locally when gRPC DDL fails)
+        try:
+            from databricks.sdk import WorkspaceClient
+            host = os.getenv("DATABRICKS_HOST")
+            token = os.getenv("DATABRICKS_TOKEN")
+            if host and token:
+                w = WorkspaceClient(host=host, token=token)
+                w.catalogs.get(catalog)
+                return True
+        except Exception:
+            pass
+
+        return False
+
     def _auto_detect_environment(self):
         """
         Iterates through 'priority_check' list from YAML.
-        Tries to verify if the catalog exists/is accessible.
+        Tries to verify if the catalog exists/is accessible via _check_catalog_access.
         Sets self.env and self.db to the first matching environment.
+        Falls back to 'default_env' from config if all checks fail (useful locally).
         """
         print("   [Config] Auto-detecting active environment...")
 
@@ -318,23 +350,28 @@ class SatisfactoEngine:
             catalog = env_config.get("catalog")
             print(f"      -> Checking access to catalog: '{catalog}' ({env_name})...")
 
-            try:
-                # Test simple d'accès au catalogue
-                # Note: spark.sql("USE catalog") change le contexte global,
-                # on préfère juste vérifier l'existence via une requête légère ou listCatalogs si dispo.
-                # Ici on tente un simple show databases dans ce catalogue.
-                self.spark.sql(f"SHOW SCHEMAS IN `{catalog}`").limit(1).collect()
-
+            if self._check_catalog_access(catalog):
                 print(f"      ✅ Success: Connected to '{catalog}'.")
                 detected_env = env_name
                 self.env = env_name.upper()
                 self.db = catalog
-                break # On s'arrête au premier qui marche
-            except Exception as e:
-                print(f"      ❌ Failed: Cannot access '{catalog}'. Error: {e}. Moving to next priority.")
+                break
+            else:
+                print(f"      ❌ Failed: Cannot access '{catalog}'. Moving to next priority.")
 
         if not detected_env:
-            raise ValueError("CRITICAL ❌ Could not connect to ANY defined environment catalog.")
+            # Fallback: use 'default_env' from config (avoids hard failure in local dev)
+            default_env = self.config.get("default_env")
+            if default_env and default_env in environments:
+                catalog = environments[default_env].get("catalog")
+                print(
+                    f"      ⚠️ [Fallback] All catalog checks failed. "
+                    f"Using 'default_env': '{default_env}' -> catalog '{catalog}'."
+                )
+                self.env = default_env.upper()
+                self.db = catalog
+            else:
+                raise ValueError("CRITICAL ❌ Could not connect to ANY defined environment catalog.")
 
     def _is_running_as_job(self):
         """
@@ -372,25 +409,44 @@ class SatisfactoEngine:
     def _get_clean_username(self):
         """
         Récupère le user courant (ex: 'julien_hou').
-        
+        Three strategies in order:
+        1. Spark SQL current_user() — works natively on Databricks.
+        2. Databricks SDK REST API — works locally when gRPC DDL fails.
+        3. DBUtils notebook context tags — fallback for classic clusters.
+
         Returns:
             str: The cleaned username of the current execution context.
         """
+        def _clean(email):
+            return email.split('@')[0].replace('.', '_').replace('-', '_').lower()
+
+        # Strategy 1: Spark SQL (works natively on Databricks)
         try:
-            # Priorité à Spark SQL (compatible VS Code & Notebooks)
             rows = self.spark.sql("SELECT current_user()").collect()
-            if rows:
-                email = rows[0][0]
-                return email.split('@')[0].replace('.', '_').replace('-', '_').lower()
-        except:
+            if rows and rows[0][0]:
+                return _clean(rows[0][0])
+        except Exception:
             pass
 
-        # Fallback Tags
+        # Strategy 2: Databricks SDK REST API (works locally when gRPC fails)
+        try:
+            from databricks.sdk import WorkspaceClient
+            host = os.getenv("DATABRICKS_HOST")
+            token = os.getenv("DATABRICKS_TOKEN")
+            if host and token:
+                w = WorkspaceClient(host=host, token=token)
+                email = w.current_user.me().user_name
+                if email:
+                    return _clean(email)
+        except Exception:
+            pass
+
+        # Strategy 3: DBUtils notebook context tags (classic clusters / notebooks)
         try:
             context = self.dbutils.notebook.entry_point.getDbutils().notebook().getContext()
             email = context.tags().apply('user')
-            return email.split('@')[0].replace('.', '_').replace('-', '_').lower()
-        except:
+            return _clean(email)
+        except Exception:
             return "unknown_user"
 
 

{satisfactoscript-0.5.5 → satisfactoscript-0.5.7}/src/satisfactoscript.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: satisfactoscript
-Version: 0.5.5
+Version: 0.5.7
 Summary: An Enterprise-Ready, Declarative Data Engineering Framework for Databricks Lakehouse.
 Author: julhouba
 Classifier: Programming Language :: Python :: 3

{satisfactoscript-0.5.5 → satisfactoscript-0.5.7}/src/satisfactoscript.egg-info/SOURCES.txt

@@ -17,6 +17,8 @@ src/satisfactoscript/semantic/__init__.py
 src/satisfactoscript/semantic/semantic.py
 tests/test_config.py
 tests/test_core.py
+tests/test_core_env_detection.py
+tests/test_core_username.py
 tests/test_dummy.py
 tests/test_loaders.py
 tests/test_registry.py

satisfactoscript-0.5.7/tests/test_core_env_detection.py

@@ -0,0 +1,159 @@
+"""
+Tests for SatisfactoEngine catalog access and environment auto-detection.
+Covers: _check_catalog_access and _auto_detect_environment.
+"""
+import pytest
+from unittest.mock import MagicMock, patch
+from satisfactoscript.core.core import SatisfactoEngine
+
+
+def _make_engine(config, spark=None):
+    """
+    Build a SatisfactoEngine instance without calling __init__,
+    allowing isolated testing of individual methods.
+    """
+    engine = object.__new__(SatisfactoEngine)
+    engine.config = config
+    engine.spark = spark or MagicMock()
+    return engine
+
+
+CONFIG_TWO_ENVS = {
+    "priority_check": ["dev", "prod"],
+    "environments": {
+        "dev":  {"catalog": "catalog_dev"},
+        "prod": {"catalog": "catalog_prd"},
+    },
+}
+
+CONFIG_WITH_DEFAULT = {
+    "priority_check": ["dev", "prod"],
+    "default_env": "dev",
+    "environments": {
+        "dev":  {"catalog": "catalog_dev"},
+        "prod": {"catalog": "catalog_prd"},
+    },
+}
+
+CONFIG_EMPTY = {}
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# _check_catalog_access
+# ──────────────────────────────────────────────────────────────────────────────
+
+class TestCheckCatalogAccess:
+
+    def test_returns_true_when_spark_sql_succeeds(self):
+        spark = MagicMock()
+        spark.sql.return_value.limit.return_value.collect.return_value = [("row",)]
+        engine = _make_engine(CONFIG_TWO_ENVS, spark=spark)
+
+        assert engine._check_catalog_access("catalog_dev") is True
+        spark.sql.assert_called_once_with("SHOW SCHEMAS IN `catalog_dev`")
+
+    def test_falls_back_to_sdk_when_spark_sql_fails(self):
+        spark = MagicMock()
+        spark.sql.side_effect = Exception("gRPC: Missing UserContext")
+        engine = _make_engine(CONFIG_TWO_ENVS, spark=spark)
+
+        mock_catalog_info = MagicMock()
+        mock_workspace_client = MagicMock()
+        mock_workspace_client.catalogs.get.return_value = mock_catalog_info
+
+        with patch.dict("os.environ", {"DATABRICKS_HOST": "https://adb-xxx.azuredatabricks.net", "DATABRICKS_TOKEN": "dapi123"}):
+            with patch("satisfactoscript.core.core.WorkspaceClient", return_value=mock_workspace_client, create=True):
+                # The SDK import inside the method needs to be patched at the right place
+                with patch.dict("sys.modules", {"databricks.sdk": MagicMock(WorkspaceClient=mock_workspace_client.__class__)}):
+                    # Patch the import inside the method
+                    import sys
+                    fake_sdk = MagicMock()
+                    fake_sdk.WorkspaceClient.return_value = mock_workspace_client
+                    sys.modules["databricks.sdk"] = fake_sdk
+
+                    result = engine._check_catalog_access("catalog_dev")
+
+        mock_workspace_client.catalogs.get.assert_called_once_with("catalog_dev")
+        assert result is True
+
+    def test_returns_false_when_both_strategies_fail(self):
+        spark = MagicMock()
+        spark.sql.side_effect = Exception("SQL error")
+        engine = _make_engine(CONFIG_TWO_ENVS, spark=spark)
+
+        with patch.dict("os.environ", {"DATABRICKS_HOST": "https://adb-xxx.azuredatabricks.net", "DATABRICKS_TOKEN": "dapi123"}):
+            import sys
+            fake_sdk = MagicMock()
+            fake_sdk.WorkspaceClient.return_value.catalogs.get.side_effect = Exception("SDK error")
+            sys.modules["databricks.sdk"] = fake_sdk
+
+            result = engine._check_catalog_access("catalog_dev")
+
+        assert result is False
+
+    def test_returns_false_when_no_credentials_for_sdk(self):
+        spark = MagicMock()
+        spark.sql.side_effect = Exception("SQL error")
+        engine = _make_engine(CONFIG_TWO_ENVS, spark=spark)
+
+        with patch.dict("os.environ", {}, clear=True):
+            result = engine._check_catalog_access("catalog_dev")
+
+        assert result is False
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# _auto_detect_environment
+# ──────────────────────────────────────────────────────────────────────────────
+
+class TestAutoDetectEnvironment:
+
+    def test_detects_first_accessible_env(self):
+        engine = _make_engine(CONFIG_TWO_ENVS)
+        engine._check_catalog_access = MagicMock(side_effect=lambda c: c == "catalog_dev")
+
+        engine._auto_detect_environment()
+
+        assert engine.env == "DEV"
+        assert engine.db == "catalog_dev"
+
+    def test_skips_inaccessible_env_and_detects_next(self):
+        engine = _make_engine(CONFIG_TWO_ENVS)
+        engine._check_catalog_access = MagicMock(side_effect=lambda c: c == "catalog_prd")
+
+        engine._auto_detect_environment()
+
+        assert engine.env == "PROD"
+        assert engine.db == "catalog_prd"
+
+    def test_uses_default_env_when_all_checks_fail(self):
+        engine = _make_engine(CONFIG_WITH_DEFAULT)
+        engine._check_catalog_access = MagicMock(return_value=False)
+
+        engine._auto_detect_environment()
+
+        assert engine.env == "DEV"
+        assert engine.db == "catalog_dev"
+
+    def test_raises_when_all_fail_and_no_default_env(self):
+        engine = _make_engine(CONFIG_TWO_ENVS)
+        engine._check_catalog_access = MagicMock(return_value=False)
+
+        with pytest.raises(ValueError, match="CRITICAL"):
+            engine._auto_detect_environment()
+
+    def test_raises_on_invalid_config(self):
+        engine = _make_engine(CONFIG_EMPTY)
+
+        with pytest.raises(ValueError, match="CRITICAL"):
+            engine._auto_detect_environment()
+
+    def test_default_env_not_used_when_sql_succeeds(self):
+        """default_env must not override a successfully detected environment."""
+        engine = _make_engine(CONFIG_WITH_DEFAULT)
+        engine._check_catalog_access = MagicMock(side_effect=lambda c: c == "catalog_prd")
+
+        engine._auto_detect_environment()
+
+        assert engine.env == "PROD"
+        assert engine.db == "catalog_prd"

satisfactoscript-0.5.7/tests/test_core_username.py

@@ -0,0 +1,101 @@
+"""
+Tests for SatisfactoEngine._get_clean_username.
+Covers: Spark SQL, Databricks SDK REST API, DBUtils fallback, unknown_user.
+"""
+import sys
+import pytest
+from unittest.mock import MagicMock, patch
+from satisfactoscript.core.core import SatisfactoEngine
+
+
+def _make_engine(spark=None, dbutils=None):
+    engine = object.__new__(SatisfactoEngine)
+    engine.spark = spark or MagicMock()
+    engine.dbutils = dbutils or MagicMock()
+    return engine
+
+
+class TestGetCleanUsername:
+
+    def test_returns_username_from_spark_sql(self):
+        spark = MagicMock()
+        spark.sql.return_value.collect.return_value = [("julien.houba@company.com",)]
+        engine = _make_engine(spark=spark)
+
+        assert engine._get_clean_username() == "julien_houba"
+
+    def test_cleans_dots_and_dashes(self):
+        spark = MagicMock()
+        spark.sql.return_value.collect.return_value = [("jean-pierre.dupont@company.com",)]
+        engine = _make_engine(spark=spark)
+
+        assert engine._get_clean_username() == "jean_pierre_dupont"
+
+    def test_falls_back_to_sdk_when_spark_sql_fails(self):
+        spark = MagicMock()
+        spark.sql.side_effect = Exception("gRPC: Missing UserContext")
+        engine = _make_engine(spark=spark)
+
+        mock_me = MagicMock()
+        mock_me.user_name = "hqhoujul@company.com"
+        mock_wc = MagicMock()
+        mock_wc.current_user.me.return_value = mock_me
+
+        fake_sdk = MagicMock()
+        fake_sdk.WorkspaceClient.return_value = mock_wc
+        sys.modules["databricks.sdk"] = fake_sdk
+
+        with patch.dict("os.environ", {"DATABRICKS_HOST": "https://adb.net", "DATABRICKS_TOKEN": "dapi123"}):
+            result = engine._get_clean_username()
+
+        assert result == "hqhoujul"
+
+    def test_falls_back_to_dbutils_when_sql_and_sdk_fail(self):
+        spark = MagicMock()
+        spark.sql.side_effect = Exception("SQL error")
+        engine = _make_engine(spark=spark)
+
+        fake_sdk = MagicMock()
+        fake_sdk.WorkspaceClient.side_effect = Exception("SDK error")
+        sys.modules["databricks.sdk"] = fake_sdk
+
+        context = MagicMock()
+        context.tags.return_value.apply.return_value = "fallback.user@company.com"
+        engine.dbutils.notebook.entry_point.getDbutils.return_value.notebook.return_value.getContext.return_value = context
+
+        with patch.dict("os.environ", {"DATABRICKS_HOST": "https://adb.net", "DATABRICKS_TOKEN": "dapi123"}):
+            result = engine._get_clean_username()
+
+        assert result == "fallback_user"
+
+    def test_returns_unknown_user_when_all_strategies_fail(self):
+        spark = MagicMock()
+        spark.sql.side_effect = Exception("SQL error")
+        engine = _make_engine(spark=spark)
+
+        fake_sdk = MagicMock()
+        fake_sdk.WorkspaceClient.side_effect = Exception("SDK error")
+        sys.modules["databricks.sdk"] = fake_sdk
+
+        engine.dbutils.notebook.entry_point.getDbutils.side_effect = Exception("no context")
+
+        with patch.dict("os.environ", {}, clear=True):
+            result = engine._get_clean_username()
+
+        assert result == "unknown_user"
+
+    def test_sdk_not_called_when_no_credentials(self):
+        spark = MagicMock()
+        spark.sql.side_effect = Exception("SQL error")
+        engine = _make_engine(spark=spark)
+
+        fake_sdk = MagicMock()
+        sys.modules["databricks.sdk"] = fake_sdk
+
+        engine.dbutils.notebook.entry_point.getDbutils.side_effect = Exception("no context")
+
+        with patch.dict("os.environ", {}, clear=True):
+            engine._get_clean_username()
+
+        # WorkspaceClient must not be called without credentials
+        fake_sdk.WorkspaceClient.assert_not_called()