dlt-utils-lib 1.2.4__tar.gz → 1.2.5__tar.gz

{dlt_utils_lib-1.2.4 → dlt_utils_lib-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.2.4
+Version: 1.2.5
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.4 → dlt_utils_lib-1.2.5}/dlt_utils/main_cdc_utils.py

@@ -33,6 +33,29 @@ def _build_table_properties(extra: Optional[dict] = None,
     return props
 
 
+def _pick_latest_source_file(spark, files_path: str) -> Optional[str]:
+    """Return the path of the most recently modified source file, or None.
+
+    Spark's parquet reader cannot mergeSchema across decimal scales
+    (e.g. decimal(19,2) vs decimal(19,3) — their integer-digit counts
+    differ). Grabbing the newest file's schema instead gives us the
+    current source-of-truth type; older files with narrower types are
+    read losslessly under that hint by Auto Loader, and Delta type
+    widening handles the existing bronze table.
+    """
+    listing = (
+        spark.read.format('binaryFile')
+        .load(files_path)
+        .select('path', 'modificationTime')
+        .orderBy('modificationTime', ascending=False)
+        .limit(1)
+        .collect()
+    )
+    if not listing:
+        return None
+    return listing[0]['path']
+
+
 # Base code pipeline: streaming process
 def base_cdc_replication_process(dlt,
                                  spark,
@@ -123,12 +146,16 @@ def create_bronze_table_definition(spark,
         table_properties=table_properties,
     )
     def transform_cdc_to_bronze():
-        # mergeSchema=true tells the parquet reader to union footers across every
-        # bronze file, so schemaHints below always reflect the widest observed
-        # type for each column (e.g. picks decimal(19,3) once the source starts
-        # writing at that scale, instead of pinning to the first-seen (19,2)
-        # and rescuing new values to NULL).
-        df = spark.read.format(file_format).option("mergeSchema", "true").load(files_path).transform(rename_columns)
+        # Read the schema from the newest source file so schemaHints tracks
+        # the current source-of-truth type after upstream drift (e.g. DMS
+        # widening decimal(19,2) -> decimal(19,3)). Directory reads —
+        # with or without mergeSchema — fail for scale-changed decimals
+        # because Spark can't merge them (integer-digit counts differ).
+        # Older files with narrower types cast up losslessly under this
+        # hint, and delta.enableTypeWidening handles the bronze table.
+        latest_file = _pick_latest_source_file(spark, files_path)
+        schema_source_path = latest_file if latest_file else files_path
+        df = spark.read.format(file_format).load(schema_source_path).transform(rename_columns)
         fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
         schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
         reader = spark \

{dlt_utils_lib-1.2.4 → dlt_utils_lib-1.2.5}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.2.4
+Version: 1.2.5
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.4 → dlt_utils_lib-1.2.5}/tests/test_main_cdc_utils.py

@@ -140,7 +140,7 @@ def test_silver_streaming_process_merges_extra_table_properties(monkeypatch):
     assert table_properties["my.custom.prop"] == "42"
 
 
-def test_create_bronze_table_definition_reads_schema_with_merge_and_enables_type_widening():
+def test_create_bronze_table_definition_reads_schema_from_latest_file_and_enables_type_widening(monkeypatch):
     class _DT:
         def __init__(self, t):
             self._t = t
@@ -164,25 +164,28 @@ def test_create_bronze_table_definition_reads_schema_with_merge_and_enables_type
         def transform(self, _fn):
             return self
 
-    calls = {"options": {}, "format": None}
+    calls = {"load_paths": [], "format": None}
 
     class _Reader:
         def format(self, fmt):
             calls["format"] = fmt
             return self
 
-        def option(self, key, value):
-            calls["options"][key] = value
-            return self
-
-        def load(self, _path):
-            # After a mergeSchema=true read the parquet reader would union
-            # decimal(19,2) and decimal(19,3) footers into decimal(19,3).
+        def load(self, path):
+            calls["load_paths"].append(path)
+            # The picked "latest" file reports decimal(19,3), the current
+            # source-of-truth after upstream drift.
             return _DF([_Field("pnl", "decimal(19,3)"), _Field("id", "bigint")])
 
     spark = type("_Spark", (), {})()
     spark.read = _Reader()
 
+    monkeypatch.setattr(
+        main_cdc_utils,
+        "_pick_latest_source_file",
+        lambda _spark, _files_path: "s3://bucket/dms_bronze/artemis/settled_bet/20260701-090901614.parquet",
+    )
+
     captured = {}
 
     class _Dlt:
@@ -218,5 +221,73 @@ def test_create_bronze_table_definition_reads_schema_with_merge_and_enables_type
         pass
 
     assert calls["format"] == "parquet"
-    assert calls["options"].get("mergeSchema") == "true"
+    assert calls["load_paths"] == [
+        "s3://bucket/dms_bronze/artemis/settled_bet/20260701-090901614.parquet"
+    ]
     assert captured["kwargs"]["table_properties"]["delta.enableTypeWidening"] == "true"
+
+
+def test_create_bronze_table_definition_falls_back_to_directory_read_when_listing_empty(monkeypatch):
+    class _DT:
+        def simpleString(self):
+            return "bigint"
+
+    class _Field:
+        def __init__(self, name):
+            self.name = name
+            self.dataType = _DT()
+
+    class _DF:
+        def __init__(self):
+            class _Schema:
+                pass
+
+            self.schema = _Schema()
+            self.schema.fields = [_Field("id")]
+
+        def transform(self, _fn):
+            return self
+
+    load_paths = []
+
+    class _Reader:
+        def format(self, _fmt):
+            return self
+
+        def load(self, path):
+            load_paths.append(path)
+            return _DF()
+
+    spark = type("_Spark", (), {})()
+    spark.read = _Reader()
+
+    monkeypatch.setattr(main_cdc_utils, "_pick_latest_source_file", lambda *_: None)
+
+    class _Dlt:
+        def table(self, **_kwargs):
+            def _decorator(fn):
+                _Dlt.fn = staticmethod(fn)
+                return fn
+
+            return _decorator
+
+    main_cdc_utils.create_bronze_table_definition(
+        spark=spark,
+        dlt=_Dlt(),
+        source_table_name="settled_bet",
+        table_name="apollo.artemis.bronze_settled_bet",
+        files_path="s3://bucket/dms_bronze/artemis/settled_bet/",
+        file_format="parquet",
+        partitions=[],
+        schema_exclude_columns=[],
+        time_diff_history_cdc_timestamp=30,
+        default_value_for_removed_col={},
+        use_managed_file_events=False,
+    )
+
+    try:
+        _Dlt.fn()
+    except AttributeError:
+        pass
+
+    assert load_paths == ["s3://bucket/dms_bronze/artemis/settled_bet/"]