dlt-utils-lib 1.2.4__tar.gz → 1.2.6__tar.gz

{dlt_utils_lib-1.2.4 → dlt_utils_lib-1.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.2.4
+Version: 1.2.6
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.4 → dlt_utils_lib-1.2.6}/dlt_utils/main_cdc_utils.py

@@ -33,6 +33,29 @@ def _build_table_properties(extra: Optional[dict] = None,
     return props
 
 
+def _pick_latest_source_file(spark, files_path: str) -> Optional[str]:
+    """Return the path of the most recently modified source file, or None.
+
+    Spark's parquet reader cannot mergeSchema across decimal scales
+    (e.g. decimal(19,2) vs decimal(19,3) — their integer-digit counts
+    differ). Grabbing the newest file's schema instead gives us the
+    current source-of-truth type; older files with narrower types are
+    read losslessly under that hint by Auto Loader, and Delta type
+    widening handles the existing bronze table.
+    """
+    listing = (
+        spark.read.format('binaryFile')
+        .load(files_path)
+        .select('path', 'modificationTime')
+        .orderBy('modificationTime', ascending=False)
+        .limit(1)
+        .collect()
+    )
+    if not listing:
+        return None
+    return listing[0]['path']
+
+
 # Base code pipeline: streaming process
 def base_cdc_replication_process(dlt,
                                  spark,
@@ -116,6 +139,27 @@ def create_bronze_table_definition(spark,
                                    ):
     table_properties = _build_table_properties(extra_table_properties)
 
+    # Resolve the schemaHints outside the @dlt.table body so that the
+    # collect() inside _pick_latest_source_file runs once at pipeline
+    # setup — Lakeflow Declarative Pipelines forbids eager Spark ops
+    # like collect() inside a decorated function.
+    #
+    # We take the schema from the newest source file so hints track the
+    # current source-of-truth type after upstream drift (e.g. DMS
+    # widening decimal(19,2) -> decimal(19,3)). Directory reads — with
+    # or without mergeSchema — fail for scale-changed decimals because
+    # Spark cannot merge them (integer-digit counts differ). Older
+    # files with narrower types cast up losslessly under this hint, and
+    # delta.enableTypeWidening handles the existing bronze table.
+    latest_file = _pick_latest_source_file(spark, files_path)
+    schema_source_path = latest_file if latest_file else files_path
+    inferred_df = spark.read.format(file_format).load(schema_source_path).transform(rename_columns)
+    schema_string = ', '.join([
+        f"{field.name} {field.dataType.simpleString()}"
+        for field in inferred_df.schema.fields
+        if field.name not in schema_exclude_columns
+    ])
+
     @dlt.table(
         name=table_name,
         comment=f"This is the bronze table for table {source_table_name}.",
@@ -123,14 +167,6 @@ def create_bronze_table_definition(spark,
         table_properties=table_properties,
     )
     def transform_cdc_to_bronze():
-        # mergeSchema=true tells the parquet reader to union footers across every
-        # bronze file, so schemaHints below always reflect the widest observed
-        # type for each column (e.g. picks decimal(19,3) once the source starts
-        # writing at that scale, instead of pinning to the first-seen (19,2)
-        # and rescuing new values to NULL).
-        df = spark.read.format(file_format).option("mergeSchema", "true").load(files_path).transform(rename_columns)
-        fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
-        schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
         reader = spark \
             .readStream \
             .format('cloudFiles') \

{dlt_utils_lib-1.2.4 → dlt_utils_lib-1.2.6}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.2.4
+Version: 1.2.6
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.4 → dlt_utils_lib-1.2.6}/tests/test_main_cdc_utils.py

@@ -33,9 +33,15 @@ def _build_cdc_table():
     }
 
 
+def _stub_bronze_setup(monkeypatch):
+    """Bypass the setup-time parquet read used to infer schemaHints."""
+    monkeypatch.setattr(main_cdc_utils, "_pick_latest_source_file", lambda *_: None)
+
+
 def test_base_cdc_replication_process_uses_pipeline_schema_configuration(monkeypatch):
     dlt_spy = _DltSpy()
-    spark = object()
+    spark = _make_spark_with_reader([], {})
+    _stub_bronze_setup(monkeypatch)
 
     monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
     monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
@@ -58,7 +64,8 @@ def test_base_cdc_replication_process_uses_pipeline_schema_configuration(monkeyp
 
 def test_base_cdc_replication_process_uses_legacy_names_without_both_schemas(monkeypatch):
     dlt_spy = _DltSpy()
-    spark = object()
+    spark = _make_spark_with_reader([], {})
+    _stub_bronze_setup(monkeypatch)
 
     monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
     monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
@@ -140,48 +147,52 @@ def test_silver_streaming_process_merges_extra_table_properties(monkeypatch):
     assert table_properties["my.custom.prop"] == "42"
 
 
-def test_create_bronze_table_definition_reads_schema_with_merge_and_enables_type_widening():
-    class _DT:
-        def __init__(self, t):
-            self._t = t
+class _FakeField:
+    def __init__(self, name, type_str):
+        self.name = name
 
-        def simpleString(self):
-            return self._t
+        class _DT:
+            def simpleString(self_inner):
+                return type_str
 
-    class _Field:
-        def __init__(self, name, type_str):
-            self.name = name
-            self.dataType = _DT(type_str)
+        self.dataType = _DT()
 
-    class _DF:
-        def __init__(self, fields):
-            class _Schema:
-                pass
 
-            self.schema = _Schema()
-            self.schema.fields = fields
+class _FakeDF:
+    def __init__(self, fields):
+        class _Schema:
+            pass
 
-        def transform(self, _fn):
-            return self
+        self.schema = _Schema()
+        self.schema.fields = fields
 
-    calls = {"options": {}, "format": None}
+    def transform(self, _fn):
+        return self
 
-    class _Reader:
-        def format(self, fmt):
-            calls["format"] = fmt
-            return self
 
-        def option(self, key, value):
-            calls["options"][key] = value
+def _make_spark_with_reader(load_paths, fields_by_path):
+    class _Reader:
+        def format(self, _fmt):
             return self
 
-        def load(self, _path):
-            # After a mergeSchema=true read the parquet reader would union
-            # decimal(19,2) and decimal(19,3) footers into decimal(19,3).
-            return _DF([_Field("pnl", "decimal(19,3)"), _Field("id", "bigint")])
+        def load(self, path):
+            load_paths.append(path)
+            return _FakeDF(fields_by_path.get(path, []))
 
     spark = type("_Spark", (), {})()
     spark.read = _Reader()
+    return spark
+
+
+def test_create_bronze_table_definition_reads_schema_from_latest_file_at_setup_time(monkeypatch):
+    load_paths = []
+    latest_path = "s3://bucket/dms_bronze/artemis/settled_bet/20260701-090901614.parquet"
+    spark = _make_spark_with_reader(
+        load_paths,
+        {latest_path: [_FakeField("pnl", "decimal(19,3)"), _FakeField("id", "bigint")]},
+    )
+
+    monkeypatch.setattr(main_cdc_utils, "_pick_latest_source_file", lambda *_: latest_path)
 
     captured = {}
 
@@ -209,14 +220,42 @@ def test_create_bronze_table_definition_reads_schema_with_merge_and_enables_type
         use_managed_file_events=False,
     )
 
-    # dlt.table just captures the transform — invoke it to exercise the
-    # initial spark.read path (the readStream chain that follows is not
-    # exercised here; the AttributeError it raises is expected).
-    try:
-        captured["fn"]()
-    except AttributeError:
-        pass
-
-    assert calls["format"] == "parquet"
-    assert calls["options"].get("mergeSchema") == "true"
+    # The schema-inference read must happen at setup time, BEFORE the
+    # @dlt.table body ever runs. That means load_paths already contains
+    # the latest file just from calling create_bronze_table_definition.
+    assert load_paths == [latest_path]
     assert captured["kwargs"]["table_properties"]["delta.enableTypeWidening"] == "true"
+
+
+def test_create_bronze_table_definition_falls_back_to_directory_when_listing_empty(monkeypatch):
+    load_paths = []
+    directory = "s3://bucket/dms_bronze/artemis/settled_bet/"
+    spark = _make_spark_with_reader(
+        load_paths,
+        {directory: [_FakeField("id", "bigint")]},
+    )
+
+    monkeypatch.setattr(main_cdc_utils, "_pick_latest_source_file", lambda *_: None)
+
+    class _Dlt:
+        def table(self, **_kwargs):
+            def _decorator(fn):
+                return fn
+
+            return _decorator
+
+    main_cdc_utils.create_bronze_table_definition(
+        spark=spark,
+        dlt=_Dlt(),
+        source_table_name="settled_bet",
+        table_name="apollo.artemis.bronze_settled_bet",
+        files_path=directory,
+        file_format="parquet",
+        partitions=[],
+        schema_exclude_columns=[],
+        time_diff_history_cdc_timestamp=30,
+        default_value_for_removed_col={},
+        use_managed_file_events=False,
+    )
+
+    assert load_paths == [directory]