dlt-utils-lib 1.2.0__tar.gz → 1.2.2__tar.gz

{dlt_utils_lib-1.2.0 → dlt_utils_lib-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.2.0
+Version: 1.2.2
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

dlt_utils_lib-1.2.2/dlt_utils/dlt_autoloader_recovery_configuration.py

@@ -0,0 +1,26 @@
+import logging
+
+from databricks.sdk import WorkspaceClient
+
+
+def _needs_autoloader_recovery(pipeline_id: str) -> bool:
+    event = next(iter(WorkspaceClient().pipelines.list_pipeline_events(
+        pipeline_id=pipeline_id, order_by=["timestamp desc"], max_results=1
+    )), None)
+    if event is None:
+        logging.warning("No pipeline events found for pipeline_id=%s", pipeline_id)
+        return False
+    if event.error:
+        for exc in event.error.exceptions or []:
+            if exc.message and "CF_MANAGED_FILE_EVENTS_INVALID_CONTINUATION_TOKEN" in exc.message:
+                return True
+    return False
+
+
+def resolve_autoloader_recovery(spark) -> bool:
+    method = spark.conf.get("autoloader_recovery_method", "AUTO").upper()
+    if method == "ON":
+        return True
+    if method == "AUTO":
+        return _needs_autoloader_recovery(spark.conf.get("pipelines.id"))
+    return False

{dlt_utils_lib-1.2.0 → dlt_utils_lib-1.2.2}/dlt_utils/main_cdc_utils.py

@@ -3,6 +3,7 @@ from typing import Optional
 
 from pyspark.sql.functions import col, expr, current_timestamp
 
+from .dlt_autoloader_recovery_configuration import resolve_autoloader_recovery
 from .dlt_transformations import (
     add_default_value_for_removed_col,
     apply_partitions,
@@ -105,6 +106,13 @@ def create_bronze_table_definition(spark,
         if use_managed_file_events:
             reader = reader.option("cloudFiles.useManagedFileEvents", "true")
 
+        if resolve_autoloader_recovery(spark):
+            reader = (
+                reader
+                .option("cloudFiles.listOnStart", "true")
+                .option("cloudFiles.validateOptions", "false")
+            )
+
         return reader.load(files_path) \
             .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp')) \
             .withColumn('ar_h_change_seq', col('ar_h_change_seq').cast('string')) \

{dlt_utils_lib-1.2.0 → dlt_utils_lib-1.2.2}/dlt_utils/main_json_utils.py

@@ -4,6 +4,8 @@ from typing import Callable, Optional
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import col, struct
 
+from .dlt_autoloader_recovery_configuration import resolve_autoloader_recovery
+
 
 def base_json_replication_process(
         dlt,
@@ -99,6 +101,13 @@ def _build_autoloader_reader(
     if use_managed_file_events:
         reader = reader.option("cloudFiles.useManagedFileEvents", "true")
 
+    if resolve_autoloader_recovery(spark):
+        reader = (
+            reader
+            .option("cloudFiles.listOnStart", "true")
+            .option("cloudFiles.validateOptions", "false")
+        )
+
     return reader.load(source_path)
 
 

{dlt_utils_lib-1.2.0 → dlt_utils_lib-1.2.2}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.2.0
+Version: 1.2.2
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.0 → dlt_utils_lib-1.2.2}/dlt_utils_lib.egg-info/SOURCES.txt

@@ -1,5 +1,6 @@
 setup.py
 dlt_utils/__init__.py
+dlt_utils/dlt_autoloader_recovery_configuration.py
 dlt_utils/dlt_metadata_receiver.py
 dlt_utils/dlt_transformations.py
 dlt_utils/main_cdc_utils.py

{dlt_utils_lib-1.2.0 → dlt_utils_lib-1.2.2}/tests/test_main_cdc_utils.py

@@ -76,3 +76,24 @@ def test_base_cdc_replication_process_uses_legacy_names_without_both_schemas(mon
     assert dlt_spy.streaming_tables[0]["name"] == "customer"
     assert dlt_spy.apply_changes_calls[0]["source"] == "bronze_customer"
     assert dlt_spy.apply_changes_calls[0]["target"] == "customer"
+
+
+def test_base_cdc_replication_process_enables_managed_file_events(monkeypatch):
+    captured_bronze_kwargs = {}
+
+    def fake_create_bronze_table_definition(**kwargs):
+        captured_bronze_kwargs.update(kwargs)
+
+    monkeypatch.setattr(main_cdc_utils, "create_bronze_table_definition", fake_create_bronze_table_definition)
+    monkeypatch.setattr(main_cdc_utils, "silver_streaming_process", lambda **kwargs: None)
+
+    main_cdc_utils.base_cdc_replication_process(
+        dlt=object(),
+        spark=object(),
+        cdc_tables_map=[_build_cdc_table()],
+        bronze_directory="dms_bronze",
+        bucket_name="s3://bucket",
+        use_managed_file_events=True,
+    )
+
+    assert captured_bronze_kwargs["use_managed_file_events"] is True

{dlt_utils_lib-1.2.0 → dlt_utils_lib-1.2.2}/tests/test_main_json_utils.py

@@ -122,3 +122,22 @@ def test_base_json_replication_process_skips_clustering_when_partitions_are_set(
     assert dlt_spy.streaming_tables[0]["partition_cols"] == ["event_date"]
     assert dlt_spy.streaming_tables[0]["cluster_by"] is None
     assert dlt_spy.streaming_tables[0]["cluster_by_auto"] is False
+
+
+def test_base_json_replication_process_enables_managed_file_events(monkeypatch):
+    captured_bronze_kwargs = {}
+
+    def fake_create_bronze_json_table_definition(**kwargs):
+        captured_bronze_kwargs.update(kwargs)
+
+    monkeypatch.setattr(main_json_utils, "create_bronze_json_table_definition", fake_create_bronze_json_table_definition)
+    monkeypatch.setattr(main_json_utils, "create_silver_json_table_definition", lambda **kwargs: None)
+
+    main_json_utils.base_json_replication_process(
+        dlt=object(),
+        spark=object(),
+        json_tables_map=[_build_json_table()],
+        use_managed_file_events=True,
+    )
+
+    assert captured_bronze_kwargs["use_managed_file_events"] is True