dlt-utils-lib 1.2.5__tar.gz → 1.2.7__tar.gz

{dlt_utils_lib-1.2.5 → dlt_utils_lib-1.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.2.5
+Version: 1.2.7
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.5 → dlt_utils_lib-1.2.7}/dlt_utils/dlt_autoloader_recovery_configuration.py

@@ -2,6 +2,8 @@ import logging
 
 from databricks.sdk import WorkspaceClient
 
+_RECOVERY_DECISION = None
+
 
 def _needs_autoloader_recovery(pipeline_id: str) -> bool:
     event = next(iter(WorkspaceClient().pipelines.list_pipeline_events(
@@ -18,9 +20,20 @@ def _needs_autoloader_recovery(pipeline_id: str) -> bool:
 
 
 def resolve_autoloader_recovery(spark) -> bool:
+    # Cached for the lifetime of the driver process: the decision belongs to the
+    # update as a whole, but callers ask once per table while the pipeline graph
+    # is built. Uncached, a 94-table pipeline makes 94 Pipelines API calls,
+    # serialized on the driver, before a single stream starts.
+    global _RECOVERY_DECISION
+    if _RECOVERY_DECISION is None:
+        _RECOVERY_DECISION = _resolve_autoloader_recovery(spark)
+    return _RECOVERY_DECISION
+
+
+def _resolve_autoloader_recovery(spark) -> bool:
     method = spark.conf.get("autoloader_recovery_method", "AUTO").upper()
     if method == "ON":
         return True
     if method == "AUTO":
         return _needs_autoloader_recovery(spark.conf.get("pipelines.id"))
-    return False
+    return False

{dlt_utils_lib-1.2.5 → dlt_utils_lib-1.2.7}/dlt_utils/main_cdc_utils.py

@@ -139,6 +139,27 @@ def create_bronze_table_definition(spark,
                                    ):
     table_properties = _build_table_properties(extra_table_properties)
 
+    # Resolve the schemaHints outside the @dlt.table body so that the
+    # collect() inside _pick_latest_source_file runs once at pipeline
+    # setup — Lakeflow Declarative Pipelines forbids eager Spark ops
+    # like collect() inside a decorated function.
+    #
+    # We take the schema from the newest source file so hints track the
+    # current source-of-truth type after upstream drift (e.g. DMS
+    # widening decimal(19,2) -> decimal(19,3)). Directory reads — with
+    # or without mergeSchema — fail for scale-changed decimals because
+    # Spark cannot merge them (integer-digit counts differ). Older
+    # files with narrower types cast up losslessly under this hint, and
+    # delta.enableTypeWidening handles the existing bronze table.
+    latest_file = _pick_latest_source_file(spark, files_path)
+    schema_source_path = latest_file if latest_file else files_path
+    inferred_df = spark.read.format(file_format).load(schema_source_path).transform(rename_columns)
+    schema_string = ', '.join([
+        f"{field.name} {field.dataType.simpleString()}"
+        for field in inferred_df.schema.fields
+        if field.name not in schema_exclude_columns
+    ])
+
     @dlt.table(
         name=table_name,
         comment=f"This is the bronze table for table {source_table_name}.",
@@ -146,18 +167,6 @@ def create_bronze_table_definition(spark,
         table_properties=table_properties,
     )
     def transform_cdc_to_bronze():
-        # Read the schema from the newest source file so schemaHints tracks
-        # the current source-of-truth type after upstream drift (e.g. DMS
-        # widening decimal(19,2) -> decimal(19,3)). Directory reads —
-        # with or without mergeSchema — fail for scale-changed decimals
-        # because Spark can't merge them (integer-digit counts differ).
-        # Older files with narrower types cast up losslessly under this
-        # hint, and delta.enableTypeWidening handles the bronze table.
-        latest_file = _pick_latest_source_file(spark, files_path)
-        schema_source_path = latest_file if latest_file else files_path
-        df = spark.read.format(file_format).load(schema_source_path).transform(rename_columns)
-        fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
-        schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
         reader = spark \
             .readStream \
             .format('cloudFiles') \

{dlt_utils_lib-1.2.5 → dlt_utils_lib-1.2.7}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.2.5
+Version: 1.2.7
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.5 → dlt_utils_lib-1.2.7}/dlt_utils_lib.egg-info/SOURCES.txt

@@ -12,6 +12,7 @@ dlt_utils_lib.egg-info/requires.txt
 dlt_utils_lib.egg-info/top_level.txt
 tests/__init__.py
 tests/conftest.py
+tests/test_autoloader_recovery_configuration.py
 tests/test_main_cdc_utils.py
 tests/test_main_json_utils.py
 tests/test_transformation_module.py

dlt_utils_lib-1.2.7/tests/test_autoloader_recovery_configuration.py

@@ -0,0 +1,68 @@
+import pytest
+
+import dlt_utils.dlt_autoloader_recovery_configuration as recovery
+
+
+class _SparkConfStub:
+    def __init__(self, values):
+        self._values = values
+
+    def get(self, key, default=None):
+        return self._values.get(key, default)
+
+
+class _SparkStub:
+    def __init__(self, **values):
+        self.conf = _SparkConfStub(values)
+
+
+@pytest.fixture(autouse=True)
+def reset_cached_decision():
+    recovery._RECOVERY_DECISION = None
+    yield
+    recovery._RECOVERY_DECISION = None
+
+
+def test_method_on_forces_recovery_without_calling_the_api(monkeypatch):
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery", _unexpected_api_call)
+
+    assert recovery.resolve_autoloader_recovery(
+        _SparkStub(autoloader_recovery_method="ON")
+    ) is True
+
+
+def test_method_off_skips_recovery_without_calling_the_api(monkeypatch):
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery", _unexpected_api_call)
+
+    assert recovery.resolve_autoloader_recovery(
+        _SparkStub(autoloader_recovery_method="OFF")
+    ) is False
+
+
+def test_method_auto_asks_the_pipelines_api(monkeypatch):
+    seen = []
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery",
+                        lambda pipeline_id: seen.append(pipeline_id) or True)
+
+    spark = _SparkStub(autoloader_recovery_method="AUTO", **{"pipelines.id": "pipeline-1"})
+
+    assert recovery.resolve_autoloader_recovery(spark) is True
+    assert seen == ["pipeline-1"]
+
+
+def test_decision_is_resolved_once_per_process(monkeypatch):
+    calls = []
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery",
+                        lambda pipeline_id: calls.append(pipeline_id) or True)
+
+    spark = _SparkStub(autoloader_recovery_method="AUTO", **{"pipelines.id": "pipeline-1"})
+
+    # One call per table while the DLT graph is built; only the first may hit the API.
+    for _ in range(10):
+        assert recovery.resolve_autoloader_recovery(spark) is True
+
+    assert calls == ["pipeline-1"]
+
+
+def _unexpected_api_call(pipeline_id):
+    raise AssertionError(f"Pipelines API should not be called (pipeline_id={pipeline_id})")

{dlt_utils_lib-1.2.5 → dlt_utils_lib-1.2.7}/tests/test_main_cdc_utils.py

@@ -33,9 +33,15 @@ def _build_cdc_table():
     }
 
 
+def _stub_bronze_setup(monkeypatch):
+    """Bypass the setup-time parquet read used to infer schemaHints."""
+    monkeypatch.setattr(main_cdc_utils, "_pick_latest_source_file", lambda *_: None)
+
+
 def test_base_cdc_replication_process_uses_pipeline_schema_configuration(monkeypatch):
     dlt_spy = _DltSpy()
-    spark = object()
+    spark = _make_spark_with_reader([], {})
+    _stub_bronze_setup(monkeypatch)
 
     monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
     monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
@@ -58,7 +64,8 @@ def test_base_cdc_replication_process_uses_pipeline_schema_configuration(monkeyp
 
 def test_base_cdc_replication_process_uses_legacy_names_without_both_schemas(monkeypatch):
     dlt_spy = _DltSpy()
-    spark = object()
+    spark = _make_spark_with_reader([], {})
+    _stub_bronze_setup(monkeypatch)
 
     monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
     monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
@@ -140,52 +147,53 @@ def test_silver_streaming_process_merges_extra_table_properties(monkeypatch):
     assert table_properties["my.custom.prop"] == "42"
 
 
-def test_create_bronze_table_definition_reads_schema_from_latest_file_and_enables_type_widening(monkeypatch):
-    class _DT:
-        def __init__(self, t):
-            self._t = t
+class _FakeField:
+    def __init__(self, name, type_str):
+        self.name = name
 
-        def simpleString(self):
-            return self._t
+        class _DT:
+            def simpleString(self_inner):
+                return type_str
 
-    class _Field:
-        def __init__(self, name, type_str):
-            self.name = name
-            self.dataType = _DT(type_str)
+        self.dataType = _DT()
 
-    class _DF:
-        def __init__(self, fields):
-            class _Schema:
-                pass
 
-            self.schema = _Schema()
-            self.schema.fields = fields
+class _FakeDF:
+    def __init__(self, fields):
+        class _Schema:
+            pass
 
-        def transform(self, _fn):
-            return self
+        self.schema = _Schema()
+        self.schema.fields = fields
 
-    calls = {"load_paths": [], "format": None}
+    def transform(self, _fn):
+        return self
 
+
+def _make_spark_with_reader(load_paths, fields_by_path):
     class _Reader:
-        def format(self, fmt):
-            calls["format"] = fmt
+        def format(self, _fmt):
             return self
 
         def load(self, path):
-            calls["load_paths"].append(path)
-            # The picked "latest" file reports decimal(19,3), the current
-            # source-of-truth after upstream drift.
-            return _DF([_Field("pnl", "decimal(19,3)"), _Field("id", "bigint")])
+            load_paths.append(path)
+            return _FakeDF(fields_by_path.get(path, []))
 
     spark = type("_Spark", (), {})()
     spark.read = _Reader()
+    return spark
+
 
-    monkeypatch.setattr(
-        main_cdc_utils,
-        "_pick_latest_source_file",
-        lambda _spark, _files_path: "s3://bucket/dms_bronze/artemis/settled_bet/20260701-090901614.parquet",
+def test_create_bronze_table_definition_reads_schema_from_latest_file_at_setup_time(monkeypatch):
+    load_paths = []
+    latest_path = "s3://bucket/dms_bronze/artemis/settled_bet/20260701-090901614.parquet"
+    spark = _make_spark_with_reader(
+        load_paths,
+        {latest_path: [_FakeField("pnl", "decimal(19,3)"), _FakeField("id", "bigint")]},
     )
 
+    monkeypatch.setattr(main_cdc_utils, "_pick_latest_source_file", lambda *_: latest_path)
+
     captured = {}
 
     class _Dlt:
@@ -212,61 +220,26 @@ def test_create_bronze_table_definition_reads_schema_from_latest_file_and_enable
         use_managed_file_events=False,
     )
 
-    # dlt.table just captures the transform — invoke it to exercise the
-    # initial spark.read path (the readStream chain that follows is not
-    # exercised here; the AttributeError it raises is expected).
-    try:
-        captured["fn"]()
-    except AttributeError:
-        pass
-
-    assert calls["format"] == "parquet"
-    assert calls["load_paths"] == [
-        "s3://bucket/dms_bronze/artemis/settled_bet/20260701-090901614.parquet"
-    ]
+    # The schema-inference read must happen at setup time, BEFORE the
+    # @dlt.table body ever runs. That means load_paths already contains
+    # the latest file just from calling create_bronze_table_definition.
+    assert load_paths == [latest_path]
     assert captured["kwargs"]["table_properties"]["delta.enableTypeWidening"] == "true"
 
 
-def test_create_bronze_table_definition_falls_back_to_directory_read_when_listing_empty(monkeypatch):
-    class _DT:
-        def simpleString(self):
-            return "bigint"
-
-    class _Field:
-        def __init__(self, name):
-            self.name = name
-            self.dataType = _DT()
-
-    class _DF:
-        def __init__(self):
-            class _Schema:
-                pass
-
-            self.schema = _Schema()
-            self.schema.fields = [_Field("id")]
-
-        def transform(self, _fn):
-            return self
-
+def test_create_bronze_table_definition_falls_back_to_directory_when_listing_empty(monkeypatch):
     load_paths = []
-
-    class _Reader:
-        def format(self, _fmt):
-            return self
-
-        def load(self, path):
-            load_paths.append(path)
-            return _DF()
-
-    spark = type("_Spark", (), {})()
-    spark.read = _Reader()
+    directory = "s3://bucket/dms_bronze/artemis/settled_bet/"
+    spark = _make_spark_with_reader(
+        load_paths,
+        {directory: [_FakeField("id", "bigint")]},
+    )
 
     monkeypatch.setattr(main_cdc_utils, "_pick_latest_source_file", lambda *_: None)
 
     class _Dlt:
         def table(self, **_kwargs):
             def _decorator(fn):
-                _Dlt.fn = staticmethod(fn)
                 return fn
 
             return _decorator
@@ -276,7 +249,7 @@ def test_create_bronze_table_definition_falls_back_to_directory_read_when_listin
         dlt=_Dlt(),
         source_table_name="settled_bet",
         table_name="apollo.artemis.bronze_settled_bet",
-        files_path="s3://bucket/dms_bronze/artemis/settled_bet/",
+        files_path=directory,
         file_format="parquet",
         partitions=[],
         schema_exclude_columns=[],
@@ -285,9 +258,4 @@ def test_create_bronze_table_definition_falls_back_to_directory_read_when_listin
         use_managed_file_events=False,
     )
 
-    try:
-        _Dlt.fn()
-    except AttributeError:
-        pass
-
-    assert load_paths == ["s3://bucket/dms_bronze/artemis/settled_bet/"]
+    assert load_paths == [directory]