satisfactoscript 0.5.8__tar.gz → 0.5.10__tar.gz

{satisfactoscript-0.5.8 → satisfactoscript-0.5.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: satisfactoscript
-Version: 0.5.8
+Version: 0.5.10
 Summary: An Enterprise-Ready, Declarative Data Engineering Framework for Databricks Lakehouse.
 Author: julhouba
 Classifier: Programming Language :: Python :: 3

{satisfactoscript-0.5.8 → satisfactoscript-0.5.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "satisfactoscript"
-version = "0.5.8"
+version = "0.5.10"
 description = "An Enterprise-Ready, Declarative Data Engineering Framework for Databricks Lakehouse."
 readme = "README.md"
 authors = [

{satisfactoscript-0.5.8 → satisfactoscript-0.5.10}/src/satisfactoscript/__init__.py

@@ -1,5 +1,6 @@
 from .core.core import SatisfactoEngine
 from .core.registry import RuleRegistry
 from .core.config import ConfigurationManager
+from .utils import safe_columns
 
-__all__ = ["SatisfactoEngine", "RuleRegistry", "ConfigurationManager"]
+__all__ = ["SatisfactoEngine", "RuleRegistry", "ConfigurationManager", "safe_columns"]

{satisfactoscript-0.5.8 → satisfactoscript-0.5.10}/src/satisfactoscript/core/core.py

@@ -657,17 +657,24 @@ class SatisfactoEngine:
         if "join" in schema_dict:
             for j in schema_dict["join"]:
                 print(f"    -> [Join] {j.get('type', 'left').upper()} JOIN {j['table_from']} -> {j['table_to']}")
-                
+
                 on_l = j["on_from"] if isinstance(j["on_from"], list) else [j["on_from"]]
                 on_r = j["on_to"] if isinstance(j["on_to"], list) else [j["on_to"]]
-                
-                cond = reduce(lambda x, y: x & y, [F.col(f"l.{l}") == F.col(f"r.{r}") for l, r in zip(on_l, on_r)])
-                
+                join_type = j.get("type", "left")
+
                 df_to = dfs[j["table_to"]]
-                cols_to_select = ["l.*"] + [F.col(f"r.{c}") for c in df_to.columns if c not in on_r]
-                
-                df_main = df_main.alias("l").join(df_to.alias("r"), cond, j.get("type", "left")) \
-                                 .select(*cols_to_select)
+
+                if on_l == on_r:
+                    # Same key names: PySpark list-join deduplicates automatically.
+                    # No schema RPC (AnalyzePlan) needed.
+                    df_main = df_main.join(df_to, on=on_l, how=join_type)
+                else:
+                    # Different key names: build condition from DataFrame column references
+                    # and drop right-side join keys post-join using unambiguous df refs.
+                    # Avoids df_to.columns (AnalyzePlan RPC) that fails with UserContext
+                    # on Databricks Connect v2 from local environments.
+                    cond = reduce(lambda x, y: x & y, [df_main[l] == df_to[r] for l, r in zip(on_l, on_r)])
+                    df_main = df_main.join(df_to, cond, join_type).drop(*[df_to[r] for r in on_r])
 
         # 3. BUSINESS RULES
         if "business_rules" in schema_dict:

satisfactoscript-0.5.10/src/satisfactoscript/registry.py

@@ -0,0 +1,7 @@
+"""
+Compatibility shim — keeps `from satisfactoscript.registry import RuleRegistry` working.
+The canonical location is satisfactoscript.core.registry.
+"""
+from satisfactoscript.core.registry import RuleRegistry
+
+__all__ = ["RuleRegistry"]

satisfactoscript-0.5.10/src/satisfactoscript/utils.py

@@ -0,0 +1,29 @@
+"""
+Utility helpers for rule authoring — compatible with Databricks Connect v2.
+"""
+
+
+def safe_columns(df):
+    """
+    Returns the column names of a DataFrame without triggering a failing AnalyzePlan RPC.
+
+    On native Databricks (cluster / notebook), this delegates to ``df.columns`` as normal.
+    On Databricks Connect v2 from local environments (PyCharm / VS Code on Windows), the
+    ``AnalyzePlan`` gRPC call that backs ``df.columns`` can fail with
+    ``Missing required field 'UserContext'``.  In that case this function returns an empty
+    list so that optional-column branches in business rules are silently skipped rather than
+    crashing.  Core transformations (withColumn, filter, write …) are unaffected because they
+    go through ``ExecutePlan``, not ``AnalyzePlan``.
+
+    Args:
+        df: A PySpark DataFrame.
+
+    Returns:
+        list[str]: Column names, or ``[]`` when schema inspection fails locally.
+    """
+    try:
+        return df.columns
+    except Exception as e:
+        if "UserContext" in str(e) or "INVALID_ARGUMENT" in str(e):
+            return []
+        raise

{satisfactoscript-0.5.8 → satisfactoscript-0.5.10}/src/satisfactoscript.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: satisfactoscript
-Version: 0.5.8
+Version: 0.5.10
 Summary: An Enterprise-Ready, Declarative Data Engineering Framework for Databricks Lakehouse.
 Author: julhouba
 Classifier: Programming Language :: Python :: 3

{satisfactoscript-0.5.8 → satisfactoscript-0.5.10}/src/satisfactoscript.egg-info/SOURCES.txt

@@ -1,6 +1,8 @@
 README.md
 pyproject.toml
 src/satisfactoscript/__init__.py
+src/satisfactoscript/registry.py
+src/satisfactoscript/utils.py
 src/satisfactoscript.egg-info/PKG-INFO
 src/satisfactoscript.egg-info/SOURCES.txt
 src/satisfactoscript.egg-info/dependency_links.txt
@@ -19,7 +21,10 @@ tests/test_config.py
 tests/test_core.py
 tests/test_core_connect_patch.py
 tests/test_core_env_detection.py
+tests/test_core_join.py
 tests/test_core_username.py
 tests/test_dummy.py
 tests/test_loaders.py
-tests/test_registry.py
+tests/test_registry.py
+tests/test_registry_import_paths.py
+tests/test_utils_safe_columns.py

satisfactoscript-0.5.10/tests/test_core_join.py

@@ -0,0 +1,140 @@
+"""
+Tests for process_schema JOIN logic.
+Key invariant: df_to.columns (AnalyzePlan gRPC) must never be called during join planning.
+"""
+import pytest
+from unittest.mock import MagicMock, call, patch
+from satisfactoscript.core.core import SatisfactoEngine
+
+
+def _make_engine():
+    engine = object.__new__(SatisfactoEngine)
+    engine.spark = MagicMock()
+    engine.config = {}
+    return engine
+
+
+def _joined_df():
+    """Return a MagicMock DataFrame that also handles chained .drop()."""
+    df = MagicMock(name="joined_df")
+    df.drop.return_value = df
+    return df
+
+
+class TestProcessSchemaJoin:
+
+    def test_same_keys_uses_list_join(self):
+        """on_from == on_to → join(df_to, on=['id'], how='inner'). No .columns call."""
+        df_a = MagicMock(name="df_a")
+        df_b = MagicMock(name="df_b")
+        joined = _joined_df()
+        df_a.join.return_value = joined
+
+        schema = {
+            "join": [{
+                "table_from": "a",
+                "table_to": "b",
+                "on_from": "id",
+                "on_to": "id",
+                "type": "inner",
+            }]
+        }
+
+        engine = _make_engine()
+        result = engine.process_schema(schema, dataframes_in={"a": df_a, "b": df_b})
+
+        df_a.join.assert_called_once_with(df_b, on=["id"], how="inner")
+        df_b.columns.assert_not_called()  # ← no AnalyzePlan gRPC
+        assert result == joined
+
+    def test_same_keys_list_join_default_left(self):
+        """Default join type is 'left'."""
+        df_a = MagicMock(name="df_a")
+        df_b = MagicMock(name="df_b")
+        df_a.join.return_value = _joined_df()
+
+        schema = {
+            "join": [{"table_from": "a", "table_to": "b", "on_from": "k", "on_to": "k"}]
+        }
+
+        engine = _make_engine()
+        engine.process_schema(schema, dataframes_in={"a": df_a, "b": df_b})
+
+        df_a.join.assert_called_once_with(df_b, on=["k"], how="left")
+
+    def test_different_keys_uses_expression_join_and_drop(self):
+        """on_from != on_to → expression join, then drop right-side keys. No .columns call."""
+        df_a = MagicMock(name="df_a")
+        df_b = MagicMock(name="df_b")
+        joined = _joined_df()
+        df_a.join.return_value = joined
+
+        schema = {
+            "join": [{
+                "table_from": "a",
+                "table_to": "b",
+                "on_from": "order_id",
+                "on_to": "id",
+                "type": "left",
+            }]
+        }
+
+        engine = _make_engine()
+        result = engine.process_schema(schema, dataframes_in={"a": df_a, "b": df_b})
+
+        # join must be called with an expression condition (not a list of strings)
+        join_args, join_kwargs = df_a.join.call_args
+        assert join_args[0] is df_b
+        assert join_args[2] == "left"
+        assert "on" not in join_kwargs  # expression-based, not list-based
+
+        # drop must be called on joined df with df_b["id"] reference
+        joined.drop.assert_called_once_with(df_b["id"])
+
+        df_b.columns.assert_not_called()  # ← no AnalyzePlan gRPC
+
+    def test_multi_key_same_names(self):
+        """Multi-column join with same key names uses list-join."""
+        df_a = MagicMock(name="df_a")
+        df_b = MagicMock(name="df_b")
+        df_a.join.return_value = _joined_df()
+
+        schema = {
+            "join": [{
+                "table_from": "a",
+                "table_to": "b",
+                "on_from": ["org", "date"],
+                "on_to": ["org", "date"],
+                "type": "left",
+            }]
+        }
+
+        engine = _make_engine()
+        engine.process_schema(schema, dataframes_in={"a": df_a, "b": df_b})
+
+        df_a.join.assert_called_once_with(df_b, on=["org", "date"], how="left")
+        df_b.columns.assert_not_called()
+
+    def test_multi_key_different_names_drops_all_right_keys(self):
+        """Multi-column join with different key names drops all right-side keys."""
+        df_a = MagicMock(name="df_a")
+        df_b = MagicMock(name="df_b")
+        joined = _joined_df()
+        df_a.join.return_value = joined
+
+        schema = {
+            "join": [{
+                "table_from": "a",
+                "table_to": "b",
+                "on_from": ["org_l", "date_l"],
+                "on_to": ["org_r", "date_r"],
+                "type": "inner",
+            }]
+        }
+
+        engine = _make_engine()
+        engine.process_schema(schema, dataframes_in={"a": df_a, "b": df_b})
+
+        # drop must receive both right-side key refs
+        joined.drop.assert_called_once_with(df_b["org_r"], df_b["date_r"])
+        df_b.columns.assert_not_called()

satisfactoscript-0.5.10/tests/test_registry_import_paths.py

@@ -0,0 +1,26 @@
+"""
+Tests for RuleRegistry import path compatibility.
+All import paths must resolve to the same singleton class.
+"""
+from satisfactoscript.core.registry import RuleRegistry as RR_core
+from satisfactoscript.registry import RuleRegistry as RR_shim
+from satisfactoscript import RuleRegistry as RR_top
+
+
+def test_all_import_paths_resolve_to_same_class():
+    """All three import paths must return the exact same class (same singleton)."""
+    assert RR_core is RR_shim
+    assert RR_core is RR_top
+
+
+def test_rule_registered_via_shim_is_visible_everywhere():
+    """A rule registered using the shim import must be retrievable from all paths."""
+    @RR_shim.register_rule(name="__test_shim_rule__")
+    def _my_rule(df):
+        return df
+
+    assert RR_core.get_rule("__test_shim_rule__") is _my_rule
+    assert RR_top.get_rule("__test_shim_rule__") is _my_rule
+
+    # Cleanup
+    RR_core._rules.pop("__test_shim_rule__", None)

satisfactoscript-0.5.10/tests/test_utils_safe_columns.py

@@ -0,0 +1,46 @@
+"""
+Tests for satisfactoscript.utils.safe_columns.
+"""
+import pytest
+from unittest.mock import MagicMock, PropertyMock
+from satisfactoscript import safe_columns
+
+
+class TestSafeColumns:
+
+    def test_returns_df_columns_normally(self):
+        df = MagicMock()
+        type(df).columns = PropertyMock(return_value=["id", "name", "value"])
+
+        assert safe_columns(df) == ["id", "name", "value"]
+
+    def test_returns_empty_list_on_usercontext_grpc_error(self):
+        df = MagicMock()
+        type(df).columns = PropertyMock(
+            side_effect=Exception("gRPC: Missing required field 'UserContext' in the request.")
+        )
+
+        result = safe_columns(df)
+
+        assert result == []
+
+    def test_returns_empty_list_on_invalid_argument_grpc_error(self):
+        df = MagicMock()
+        type(df).columns = PropertyMock(
+            side_effect=Exception("StatusCode.INVALID_ARGUMENT details: something")
+        )
+
+        result = safe_columns(df)
+
+        assert result == []
+
+    def test_reraises_unrelated_exceptions(self):
+        df = MagicMock()
+        type(df).columns = PropertyMock(side_effect=RuntimeError("disk full"))
+
+        with pytest.raises(RuntimeError, match="disk full"):
+            safe_columns(df)
+
+    def test_importable_from_satisfactoscript_top_level(self):
+        from satisfactoscript import safe_columns as sc
+        assert callable(sc)