dlt-utils-lib 1.2.6__tar.gz → 1.2.7__tar.gz

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.2.6
+Version: 1.2.7
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.2.7}/dlt_utils/dlt_autoloader_recovery_configuration.py

@@ -2,6 +2,8 @@ import logging
 
 from databricks.sdk import WorkspaceClient
 
+_RECOVERY_DECISION = None
+
 
 def _needs_autoloader_recovery(pipeline_id: str) -> bool:
     event = next(iter(WorkspaceClient().pipelines.list_pipeline_events(
@@ -18,9 +20,20 @@ def _needs_autoloader_recovery(pipeline_id: str) -> bool:
 
 
 def resolve_autoloader_recovery(spark) -> bool:
+    # Cached for the lifetime of the driver process: the decision belongs to the
+    # update as a whole, but callers ask once per table while the pipeline graph
+    # is built. Uncached, a 94-table pipeline makes 94 Pipelines API calls,
+    # serialized on the driver, before a single stream starts.
+    global _RECOVERY_DECISION
+    if _RECOVERY_DECISION is None:
+        _RECOVERY_DECISION = _resolve_autoloader_recovery(spark)
+    return _RECOVERY_DECISION
+
+
+def _resolve_autoloader_recovery(spark) -> bool:
     method = spark.conf.get("autoloader_recovery_method", "AUTO").upper()
     if method == "ON":
         return True
     if method == "AUTO":
         return _needs_autoloader_recovery(spark.conf.get("pipelines.id"))
-    return False
+    return False

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.2.7}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.2.6
+Version: 1.2.7
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.2.7}/dlt_utils_lib.egg-info/SOURCES.txt

@@ -12,6 +12,7 @@ dlt_utils_lib.egg-info/requires.txt
 dlt_utils_lib.egg-info/top_level.txt
 tests/__init__.py
 tests/conftest.py
+tests/test_autoloader_recovery_configuration.py
 tests/test_main_cdc_utils.py
 tests/test_main_json_utils.py
 tests/test_transformation_module.py

dlt_utils_lib-1.2.7/tests/test_autoloader_recovery_configuration.py

@@ -0,0 +1,68 @@
+import pytest
+
+import dlt_utils.dlt_autoloader_recovery_configuration as recovery
+
+
+class _SparkConfStub:
+    def __init__(self, values):
+        self._values = values
+
+    def get(self, key, default=None):
+        return self._values.get(key, default)
+
+
+class _SparkStub:
+    def __init__(self, **values):
+        self.conf = _SparkConfStub(values)
+
+
+@pytest.fixture(autouse=True)
+def reset_cached_decision():
+    recovery._RECOVERY_DECISION = None
+    yield
+    recovery._RECOVERY_DECISION = None
+
+
+def test_method_on_forces_recovery_without_calling_the_api(monkeypatch):
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery", _unexpected_api_call)
+
+    assert recovery.resolve_autoloader_recovery(
+        _SparkStub(autoloader_recovery_method="ON")
+    ) is True
+
+
+def test_method_off_skips_recovery_without_calling_the_api(monkeypatch):
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery", _unexpected_api_call)
+
+    assert recovery.resolve_autoloader_recovery(
+        _SparkStub(autoloader_recovery_method="OFF")
+    ) is False
+
+
+def test_method_auto_asks_the_pipelines_api(monkeypatch):
+    seen = []
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery",
+                        lambda pipeline_id: seen.append(pipeline_id) or True)
+
+    spark = _SparkStub(autoloader_recovery_method="AUTO", **{"pipelines.id": "pipeline-1"})
+
+    assert recovery.resolve_autoloader_recovery(spark) is True
+    assert seen == ["pipeline-1"]
+
+
+def test_decision_is_resolved_once_per_process(monkeypatch):
+    calls = []
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery",
+                        lambda pipeline_id: calls.append(pipeline_id) or True)
+
+    spark = _SparkStub(autoloader_recovery_method="AUTO", **{"pipelines.id": "pipeline-1"})
+
+    # One call per table while the DLT graph is built; only the first may hit the API.
+    for _ in range(10):
+        assert recovery.resolve_autoloader_recovery(spark) is True
+
+    assert calls == ["pipeline-1"]
+
+
+def _unexpected_api_call(pipeline_id):
+    raise AssertionError(f"Pipelines API should not be called (pipeline_id={pipeline_id})")