dlt-utils-lib 1.2.3__tar.gz → 1.2.5__tar.gz

{dlt_utils_lib-1.2.3 → dlt_utils_lib-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.2.3
+Version: 1.2.5
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.3 → dlt_utils_lib-1.2.5}/dlt_utils/main_cdc_utils.py

@@ -10,6 +10,52 @@ from .dlt_transformations import (
     rename_columns,
     update_cdc_timestamp,
 )
+
+
+# delta.enableTypeWidening lets Delta accept widened column types (e.g. decimal
+# scale bumps) from upstream schema drift without a manual ALTER + full refresh.
+# Applied to every bronze and silver table so future source-side type changes
+# flow through the pipeline dynamically.
+_DEFAULT_TABLE_PROPERTIES = {
+    "delta.autoOptimize.optimizeWrite": "true",
+    "delta.autoOptimize.autoCompact": "true",
+    "delta.enableTypeWidening": "true",
+}
+
+
+def _build_table_properties(extra: Optional[dict] = None,
+                            column_comments: Optional[dict] = None) -> dict:
+    props = dict(_DEFAULT_TABLE_PROPERTIES)
+    if column_comments is not None:
+        props["column_comments"] = json.dumps(column_comments) if column_comments else "{}"
+    if extra:
+        props.update(extra)
+    return props
+
+
+def _pick_latest_source_file(spark, files_path: str) -> Optional[str]:
+    """Return the path of the most recently modified source file, or None.
+
+    Spark's parquet reader cannot mergeSchema across decimal scales
+    (e.g. decimal(19,2) vs decimal(19,3) — their integer-digit counts
+    differ). Grabbing the newest file's schema instead gives us the
+    current source-of-truth type; older files with narrower types are
+    read losslessly under that hint by Auto Loader, and Delta type
+    widening handles the existing bronze table.
+    """
+    listing = (
+        spark.read.format('binaryFile')
+        .load(files_path)
+        .select('path', 'modificationTime')
+        .orderBy('modificationTime', ascending=False)
+        .limit(1)
+        .collect()
+    )
+    if not listing:
+        return None
+    return listing[0]['path']
+
+
 # Base code pipeline: streaming process
 def base_cdc_replication_process(dlt,
                                  spark,
@@ -38,6 +84,7 @@ def base_cdc_replication_process(dlt,
         table_description = table.get('table_description')
         column_comments = table.get('column_comments')
         partition_col = table.get('partition_col')
+        extra_table_properties = table.get('table_properties') or {}
 
         if bronze_schema and silver_schema:
             bronze_table_name = f"{bronze_schema}.{source_table_name}"
@@ -58,6 +105,7 @@ def base_cdc_replication_process(dlt,
                                        time_diff_history_cdc_timestamp=time_diff_history_cdc_timestamp,
                                        default_value_for_removed_col=default_value_for_removed_col,
                                        use_managed_file_events=use_managed_file_events,
+                                       extra_table_properties=extra_table_properties,
                                        )
 
         silver_streaming_process(dlt=dlt,
@@ -70,7 +118,8 @@ def base_cdc_replication_process(dlt,
                                  table_description=table_description,
                                  column_comments=column_comments,
                                  cluster_by_value=cluster_by_value,
-                                 is_cluster_by_auto=is_cluster_by_auto
+                                 is_cluster_by_auto=is_cluster_by_auto,
+                                 extra_table_properties=extra_table_properties,
                                  )
 
 
@@ -86,14 +135,27 @@ def create_bronze_table_definition(spark,
                                    time_diff_history_cdc_timestamp: int,
                                    default_value_for_removed_col: dict,
                                    use_managed_file_events: bool,
+                                   extra_table_properties: Optional[dict] = None,
                                    ):
+    table_properties = _build_table_properties(extra_table_properties)
+
     @dlt.table(
         name=table_name,
         comment=f"This is the bronze table for table {source_table_name}.",
-        temporary=False
+        temporary=False,
+        table_properties=table_properties,
     )
     def transform_cdc_to_bronze():
-        df = spark.read.format(file_format).load(files_path).transform(rename_columns)
+        # Read the schema from the newest source file so schemaHints tracks
+        # the current source-of-truth type after upstream drift (e.g. DMS
+        # widening decimal(19,2) -> decimal(19,3)). Directory reads —
+        # with or without mergeSchema — fail for scale-changed decimals
+        # because Spark can't merge them (integer-digit counts differ).
+        # Older files with narrower types cast up losslessly under this
+        # hint, and delta.enableTypeWidening handles the bronze table.
+        latest_file = _pick_latest_source_file(spark, files_path)
+        schema_source_path = latest_file if latest_file else files_path
+        df = spark.read.format(file_format).load(schema_source_path).transform(rename_columns)
         fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
         schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
         reader = spark \
@@ -135,14 +197,16 @@ def silver_streaming_process(dlt,
                              column_comments: dict = None,
                              enable_truncate: bool = False,
                              cluster_by_value: list = None,
-                             is_cluster_by_auto: bool = True):
+                             is_cluster_by_auto: bool = True,
+                             extra_table_properties: Optional[dict] = None):
+    table_properties = _build_table_properties(
+        extra_table_properties,
+        column_comments=column_comments,
+    )
+
     dlt.create_streaming_table(
         name=table_name,
-        table_properties={
-            "delta.autoOptimize.optimizeWrite": "true",
-            "delta.autoOptimize.autoCompact": "true",
-            "column_comments": json.dumps(column_comments) if column_comments else "{}"
-        },
+        table_properties=table_properties,
         comment=table_description or "This is the silver table with source in",
         partition_cols=partitions if partitions else None,
         cluster_by=cluster_by_value,

{dlt_utils_lib-1.2.3 → dlt_utils_lib-1.2.5}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.2.3
+Version: 1.2.5
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

dlt_utils_lib-1.2.5/tests/test_main_cdc_utils.py

@@ -0,0 +1,293 @@
+import dlt_utils.main_cdc_utils as main_cdc_utils
+
+
+class _DltSpy:
+    def __init__(self):
+        self.decorated_tables = []
+        self.streaming_tables = []
+        self.apply_changes_calls = []
+
+    def table(self, **kwargs):
+        self.decorated_tables.append(kwargs)
+
+        def decorator(func):
+            return func
+
+        return decorator
+
+    def create_streaming_table(self, **kwargs):
+        self.streaming_tables.append(kwargs)
+
+    def apply_changes(self, **kwargs):
+        self.apply_changes_calls.append(kwargs)
+
+
+def _build_cdc_table():
+    return {
+        "source_table_name": "customer",
+        "source_schema": "bifrost",
+        "format": "parquet",
+        "keys": ["id"],
+        "exclude_columns": [],
+        "enable_truncate": False,
+    }
+
+
+def test_base_cdc_replication_process_uses_pipeline_schema_configuration(monkeypatch):
+    dlt_spy = _DltSpy()
+    spark = object()
+
+    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
+    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
+
+    main_cdc_utils.base_cdc_replication_process(
+        dlt=dlt_spy,
+        spark=spark,
+        cdc_tables_map=[_build_cdc_table()],
+        bronze_directory="dms_bronze",
+        bucket_name="s3://bucket",
+        bronze_schema="bifrost_bronze",
+        silver_schema="bifrost_silver",
+    )
+
+    assert dlt_spy.decorated_tables[0]["name"] == "bifrost_bronze.customer"
+    assert dlt_spy.streaming_tables[0]["name"] == "bifrost_silver.customer"
+    assert dlt_spy.apply_changes_calls[0]["source"] == "bifrost_bronze.customer"
+    assert dlt_spy.apply_changes_calls[0]["target"] == "bifrost_silver.customer"
+
+
+def test_base_cdc_replication_process_uses_legacy_names_without_both_schemas(monkeypatch):
+    dlt_spy = _DltSpy()
+    spark = object()
+
+    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
+    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
+
+    main_cdc_utils.base_cdc_replication_process(
+        dlt=dlt_spy,
+        spark=spark,
+        cdc_tables_map=[_build_cdc_table()],
+        bronze_directory="dms_bronze",
+        bucket_name="s3://bucket",
+        bronze_schema="bifrost_bronze",
+    )
+
+    assert dlt_spy.decorated_tables[0]["name"] == "bronze_customer"
+    assert dlt_spy.streaming_tables[0]["name"] == "customer"
+    assert dlt_spy.apply_changes_calls[0]["source"] == "bronze_customer"
+    assert dlt_spy.apply_changes_calls[0]["target"] == "customer"
+
+
+def test_base_cdc_replication_process_enables_managed_file_events(monkeypatch):
+    captured_bronze_kwargs = {}
+
+    def fake_create_bronze_table_definition(**kwargs):
+        captured_bronze_kwargs.update(kwargs)
+
+    monkeypatch.setattr(main_cdc_utils, "create_bronze_table_definition", fake_create_bronze_table_definition)
+    monkeypatch.setattr(main_cdc_utils, "silver_streaming_process", lambda **kwargs: None)
+
+    main_cdc_utils.base_cdc_replication_process(
+        dlt=object(),
+        spark=object(),
+        cdc_tables_map=[_build_cdc_table()],
+        bronze_directory="dms_bronze",
+        bucket_name="s3://bucket",
+        use_managed_file_events=True,
+    )
+
+    assert captured_bronze_kwargs["use_managed_file_events"] is True
+
+
+def test_silver_streaming_process_enables_type_widening_by_default(monkeypatch):
+    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
+    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
+
+    dlt_spy = _DltSpy()
+    main_cdc_utils.silver_streaming_process(
+        dlt=dlt_spy,
+        table_name="apollo.artemis.settled_bet",
+        bronze_table_name="apollo.artemis.bronze_settled_bet",
+        keys=["id"],
+        partitions=[],
+        exclude_columns=[],
+    )
+
+    table_properties = dlt_spy.streaming_tables[0]["table_properties"]
+    assert table_properties["delta.enableTypeWidening"] == "true"
+    assert table_properties["delta.autoOptimize.optimizeWrite"] == "true"
+    assert table_properties["delta.autoOptimize.autoCompact"] == "true"
+
+
+def test_silver_streaming_process_merges_extra_table_properties(monkeypatch):
+    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
+    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
+
+    dlt_spy = _DltSpy()
+    main_cdc_utils.silver_streaming_process(
+        dlt=dlt_spy,
+        table_name="apollo.artemis.settled_bet",
+        bronze_table_name="apollo.artemis.bronze_settled_bet",
+        keys=["id"],
+        partitions=[],
+        exclude_columns=[],
+        extra_table_properties={"delta.enableTypeWidening": "false", "my.custom.prop": "42"},
+    )
+
+    table_properties = dlt_spy.streaming_tables[0]["table_properties"]
+    # Caller override wins over the library default.
+    assert table_properties["delta.enableTypeWidening"] == "false"
+    assert table_properties["my.custom.prop"] == "42"
+
+
+def test_create_bronze_table_definition_reads_schema_from_latest_file_and_enables_type_widening(monkeypatch):
+    class _DT:
+        def __init__(self, t):
+            self._t = t
+
+        def simpleString(self):
+            return self._t
+
+    class _Field:
+        def __init__(self, name, type_str):
+            self.name = name
+            self.dataType = _DT(type_str)
+
+    class _DF:
+        def __init__(self, fields):
+            class _Schema:
+                pass
+
+            self.schema = _Schema()
+            self.schema.fields = fields
+
+        def transform(self, _fn):
+            return self
+
+    calls = {"load_paths": [], "format": None}
+
+    class _Reader:
+        def format(self, fmt):
+            calls["format"] = fmt
+            return self
+
+        def load(self, path):
+            calls["load_paths"].append(path)
+            # The picked "latest" file reports decimal(19,3), the current
+            # source-of-truth after upstream drift.
+            return _DF([_Field("pnl", "decimal(19,3)"), _Field("id", "bigint")])
+
+    spark = type("_Spark", (), {})()
+    spark.read = _Reader()
+
+    monkeypatch.setattr(
+        main_cdc_utils,
+        "_pick_latest_source_file",
+        lambda _spark, _files_path: "s3://bucket/dms_bronze/artemis/settled_bet/20260701-090901614.parquet",
+    )
+
+    captured = {}
+
+    class _Dlt:
+        def table(self, **kwargs):
+            captured["kwargs"] = kwargs
+
+            def _decorator(fn):
+                captured["fn"] = fn
+                return fn
+
+            return _decorator
+
+    main_cdc_utils.create_bronze_table_definition(
+        spark=spark,
+        dlt=_Dlt(),
+        source_table_name="settled_bet",
+        table_name="apollo.artemis.bronze_settled_bet",
+        files_path="s3://bucket/dms_bronze/artemis/settled_bet/",
+        file_format="parquet",
+        partitions=[],
+        schema_exclude_columns=[],
+        time_diff_history_cdc_timestamp=30,
+        default_value_for_removed_col={},
+        use_managed_file_events=False,
+    )
+
+    # dlt.table just captures the transform — invoke it to exercise the
+    # initial spark.read path (the readStream chain that follows is not
+    # exercised here; the AttributeError it raises is expected).
+    try:
+        captured["fn"]()
+    except AttributeError:
+        pass
+
+    assert calls["format"] == "parquet"
+    assert calls["load_paths"] == [
+        "s3://bucket/dms_bronze/artemis/settled_bet/20260701-090901614.parquet"
+    ]
+    assert captured["kwargs"]["table_properties"]["delta.enableTypeWidening"] == "true"
+
+
+def test_create_bronze_table_definition_falls_back_to_directory_read_when_listing_empty(monkeypatch):
+    class _DT:
+        def simpleString(self):
+            return "bigint"
+
+    class _Field:
+        def __init__(self, name):
+            self.name = name
+            self.dataType = _DT()
+
+    class _DF:
+        def __init__(self):
+            class _Schema:
+                pass
+
+            self.schema = _Schema()
+            self.schema.fields = [_Field("id")]
+
+        def transform(self, _fn):
+            return self
+
+    load_paths = []
+
+    class _Reader:
+        def format(self, _fmt):
+            return self
+
+        def load(self, path):
+            load_paths.append(path)
+            return _DF()
+
+    spark = type("_Spark", (), {})()
+    spark.read = _Reader()
+
+    monkeypatch.setattr(main_cdc_utils, "_pick_latest_source_file", lambda *_: None)
+
+    class _Dlt:
+        def table(self, **_kwargs):
+            def _decorator(fn):
+                _Dlt.fn = staticmethod(fn)
+                return fn
+
+            return _decorator
+
+    main_cdc_utils.create_bronze_table_definition(
+        spark=spark,
+        dlt=_Dlt(),
+        source_table_name="settled_bet",
+        table_name="apollo.artemis.bronze_settled_bet",
+        files_path="s3://bucket/dms_bronze/artemis/settled_bet/",
+        file_format="parquet",
+        partitions=[],
+        schema_exclude_columns=[],
+        time_diff_history_cdc_timestamp=30,
+        default_value_for_removed_col={},
+        use_managed_file_events=False,
+    )
+
+    try:
+        _Dlt.fn()
+    except AttributeError:
+        pass
+
+    assert load_paths == ["s3://bucket/dms_bronze/artemis/settled_bet/"]

dlt_utils_lib-1.2.3/tests/test_main_cdc_utils.py

@@ -1,99 +0,0 @@
-import dlt_utils.main_cdc_utils as main_cdc_utils
-
-
-class _DltSpy:
-    def __init__(self):
-        self.decorated_tables = []
-        self.streaming_tables = []
-        self.apply_changes_calls = []
-
-    def table(self, **kwargs):
-        self.decorated_tables.append(kwargs)
-
-        def decorator(func):
-            return func
-
-        return decorator
-
-    def create_streaming_table(self, **kwargs):
-        self.streaming_tables.append(kwargs)
-
-    def apply_changes(self, **kwargs):
-        self.apply_changes_calls.append(kwargs)
-
-
-def _build_cdc_table():
-    return {
-        "source_table_name": "customer",
-        "source_schema": "bifrost",
-        "format": "parquet",
-        "keys": ["id"],
-        "exclude_columns": [],
-        "enable_truncate": False,
-    }
-
-
-def test_base_cdc_replication_process_uses_pipeline_schema_configuration(monkeypatch):
-    dlt_spy = _DltSpy()
-    spark = object()
-
-    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
-    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
-
-    main_cdc_utils.base_cdc_replication_process(
-        dlt=dlt_spy,
-        spark=spark,
-        cdc_tables_map=[_build_cdc_table()],
-        bronze_directory="dms_bronze",
-        bucket_name="s3://bucket",
-        bronze_schema="bifrost_bronze",
-        silver_schema="bifrost_silver",
-    )
-
-    assert dlt_spy.decorated_tables[0]["name"] == "bifrost_bronze.customer"
-    assert dlt_spy.streaming_tables[0]["name"] == "bifrost_silver.customer"
-    assert dlt_spy.apply_changes_calls[0]["source"] == "bifrost_bronze.customer"
-    assert dlt_spy.apply_changes_calls[0]["target"] == "bifrost_silver.customer"
-
-
-def test_base_cdc_replication_process_uses_legacy_names_without_both_schemas(monkeypatch):
-    dlt_spy = _DltSpy()
-    spark = object()
-
-    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
-    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
-
-    main_cdc_utils.base_cdc_replication_process(
-        dlt=dlt_spy,
-        spark=spark,
-        cdc_tables_map=[_build_cdc_table()],
-        bronze_directory="dms_bronze",
-        bucket_name="s3://bucket",
-        bronze_schema="bifrost_bronze",
-    )
-
-    assert dlt_spy.decorated_tables[0]["name"] == "bronze_customer"
-    assert dlt_spy.streaming_tables[0]["name"] == "customer"
-    assert dlt_spy.apply_changes_calls[0]["source"] == "bronze_customer"
-    assert dlt_spy.apply_changes_calls[0]["target"] == "customer"
-
-
-def test_base_cdc_replication_process_enables_managed_file_events(monkeypatch):
-    captured_bronze_kwargs = {}
-
-    def fake_create_bronze_table_definition(**kwargs):
-        captured_bronze_kwargs.update(kwargs)
-
-    monkeypatch.setattr(main_cdc_utils, "create_bronze_table_definition", fake_create_bronze_table_definition)
-    monkeypatch.setattr(main_cdc_utils, "silver_streaming_process", lambda **kwargs: None)
-
-    main_cdc_utils.base_cdc_replication_process(
-        dlt=object(),
-        spark=object(),
-        cdc_tables_map=[_build_cdc_table()],
-        bronze_directory="dms_bronze",
-        bucket_name="s3://bucket",
-        use_managed_file_events=True,
-    )
-
-    assert captured_bronze_kwargs["use_managed_file_events"] is True