dlt-utils-lib 1.2.2__tar.gz → 1.2.4__tar.gz

{dlt_utils_lib-1.2.2 → dlt_utils_lib-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.2.2
+Version: 1.2.4
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.2 → dlt_utils_lib-1.2.4}/dlt_utils/main_cdc_utils.py

@@ -10,6 +10,29 @@ from .dlt_transformations import (
     rename_columns,
     update_cdc_timestamp,
 )
+
+
+# delta.enableTypeWidening lets Delta accept widened column types (e.g. decimal
+# scale bumps) from upstream schema drift without a manual ALTER + full refresh.
+# Applied to every bronze and silver table so future source-side type changes
+# flow through the pipeline dynamically.
+_DEFAULT_TABLE_PROPERTIES = {
+    "delta.autoOptimize.optimizeWrite": "true",
+    "delta.autoOptimize.autoCompact": "true",
+    "delta.enableTypeWidening": "true",
+}
+
+
+def _build_table_properties(extra: Optional[dict] = None,
+                            column_comments: Optional[dict] = None) -> dict:
+    props = dict(_DEFAULT_TABLE_PROPERTIES)
+    if column_comments is not None:
+        props["column_comments"] = json.dumps(column_comments) if column_comments else "{}"
+    if extra:
+        props.update(extra)
+    return props
+
+
 # Base code pipeline: streaming process
 def base_cdc_replication_process(dlt,
                                  spark,
@@ -38,6 +61,7 @@ def base_cdc_replication_process(dlt,
         table_description = table.get('table_description')
         column_comments = table.get('column_comments')
         partition_col = table.get('partition_col')
+        extra_table_properties = table.get('table_properties') or {}
 
         if bronze_schema and silver_schema:
             bronze_table_name = f"{bronze_schema}.{source_table_name}"
@@ -58,6 +82,7 @@ def base_cdc_replication_process(dlt,
                                        time_diff_history_cdc_timestamp=time_diff_history_cdc_timestamp,
                                        default_value_for_removed_col=default_value_for_removed_col,
                                        use_managed_file_events=use_managed_file_events,
+                                       extra_table_properties=extra_table_properties,
                                        )
 
         silver_streaming_process(dlt=dlt,
@@ -70,7 +95,8 @@ def base_cdc_replication_process(dlt,
                                  table_description=table_description,
                                  column_comments=column_comments,
                                  cluster_by_value=cluster_by_value,
-                                 is_cluster_by_auto=is_cluster_by_auto
+                                 is_cluster_by_auto=is_cluster_by_auto,
+                                 extra_table_properties=extra_table_properties,
                                  )
 
 
@@ -86,14 +112,23 @@ def create_bronze_table_definition(spark,
                                    time_diff_history_cdc_timestamp: int,
                                    default_value_for_removed_col: dict,
                                    use_managed_file_events: bool,
+                                   extra_table_properties: Optional[dict] = None,
                                    ):
+    table_properties = _build_table_properties(extra_table_properties)
+
     @dlt.table(
         name=table_name,
         comment=f"This is the bronze table for table {source_table_name}.",
-        temporary=False
+        temporary=False,
+        table_properties=table_properties,
     )
     def transform_cdc_to_bronze():
-        df = spark.read.format(file_format).load(files_path).transform(rename_columns)
+        # mergeSchema=true tells the parquet reader to union footers across every
+        # bronze file, so schemaHints below always reflect the widest observed
+        # type for each column (e.g. picks decimal(19,3) once the source starts
+        # writing at that scale, instead of pinning to the first-seen (19,2)
+        # and rescuing new values to NULL).
+        df = spark.read.format(file_format).option("mergeSchema", "true").load(files_path).transform(rename_columns)
         fields = [field for field in df.schema.fields if field.name not in schema_exclude_columns]
         schema_string = ', '.join([f"{field.name} {field.dataType.simpleString()}" for field in fields])
         reader = spark \
@@ -135,14 +170,16 @@ def silver_streaming_process(dlt,
                              column_comments: dict = None,
                              enable_truncate: bool = False,
                              cluster_by_value: list = None,
-                             is_cluster_by_auto: bool = True):
+                             is_cluster_by_auto: bool = True,
+                             extra_table_properties: Optional[dict] = None):
+    table_properties = _build_table_properties(
+        extra_table_properties,
+        column_comments=column_comments,
+    )
+
     dlt.create_streaming_table(
         name=table_name,
-        table_properties={
-            "delta.autoOptimize.optimizeWrite": "true",
-            "delta.autoOptimize.autoCompact": "true",
-            "column_comments": json.dumps(column_comments) if column_comments else "{}"
-        },
+        table_properties=table_properties,
         comment=table_description or "This is the silver table with source in",
         partition_cols=partitions if partitions else None,
         cluster_by=cluster_by_value,

{dlt_utils_lib-1.2.2 → dlt_utils_lib-1.2.4}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.2.2
+Version: 1.2.4
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

dlt_utils_lib-1.2.4/tests/test_main_cdc_utils.py

@@ -0,0 +1,222 @@
+import dlt_utils.main_cdc_utils as main_cdc_utils
+
+
+class _DltSpy:
+    def __init__(self):
+        self.decorated_tables = []
+        self.streaming_tables = []
+        self.apply_changes_calls = []
+
+    def table(self, **kwargs):
+        self.decorated_tables.append(kwargs)
+
+        def decorator(func):
+            return func
+
+        return decorator
+
+    def create_streaming_table(self, **kwargs):
+        self.streaming_tables.append(kwargs)
+
+    def apply_changes(self, **kwargs):
+        self.apply_changes_calls.append(kwargs)
+
+
+def _build_cdc_table():
+    return {
+        "source_table_name": "customer",
+        "source_schema": "bifrost",
+        "format": "parquet",
+        "keys": ["id"],
+        "exclude_columns": [],
+        "enable_truncate": False,
+    }
+
+
+def test_base_cdc_replication_process_uses_pipeline_schema_configuration(monkeypatch):
+    dlt_spy = _DltSpy()
+    spark = object()
+
+    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
+    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
+
+    main_cdc_utils.base_cdc_replication_process(
+        dlt=dlt_spy,
+        spark=spark,
+        cdc_tables_map=[_build_cdc_table()],
+        bronze_directory="dms_bronze",
+        bucket_name="s3://bucket",
+        bronze_schema="bifrost_bronze",
+        silver_schema="bifrost_silver",
+    )
+
+    assert dlt_spy.decorated_tables[0]["name"] == "bifrost_bronze.customer"
+    assert dlt_spy.streaming_tables[0]["name"] == "bifrost_silver.customer"
+    assert dlt_spy.apply_changes_calls[0]["source"] == "bifrost_bronze.customer"
+    assert dlt_spy.apply_changes_calls[0]["target"] == "bifrost_silver.customer"
+
+
+def test_base_cdc_replication_process_uses_legacy_names_without_both_schemas(monkeypatch):
+    dlt_spy = _DltSpy()
+    spark = object()
+
+    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
+    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
+
+    main_cdc_utils.base_cdc_replication_process(
+        dlt=dlt_spy,
+        spark=spark,
+        cdc_tables_map=[_build_cdc_table()],
+        bronze_directory="dms_bronze",
+        bucket_name="s3://bucket",
+        bronze_schema="bifrost_bronze",
+    )
+
+    assert dlt_spy.decorated_tables[0]["name"] == "bronze_customer"
+    assert dlt_spy.streaming_tables[0]["name"] == "customer"
+    assert dlt_spy.apply_changes_calls[0]["source"] == "bronze_customer"
+    assert dlt_spy.apply_changes_calls[0]["target"] == "customer"
+
+
+def test_base_cdc_replication_process_enables_managed_file_events(monkeypatch):
+    captured_bronze_kwargs = {}
+
+    def fake_create_bronze_table_definition(**kwargs):
+        captured_bronze_kwargs.update(kwargs)
+
+    monkeypatch.setattr(main_cdc_utils, "create_bronze_table_definition", fake_create_bronze_table_definition)
+    monkeypatch.setattr(main_cdc_utils, "silver_streaming_process", lambda **kwargs: None)
+
+    main_cdc_utils.base_cdc_replication_process(
+        dlt=object(),
+        spark=object(),
+        cdc_tables_map=[_build_cdc_table()],
+        bronze_directory="dms_bronze",
+        bucket_name="s3://bucket",
+        use_managed_file_events=True,
+    )
+
+    assert captured_bronze_kwargs["use_managed_file_events"] is True
+
+
+def test_silver_streaming_process_enables_type_widening_by_default(monkeypatch):
+    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
+    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
+
+    dlt_spy = _DltSpy()
+    main_cdc_utils.silver_streaming_process(
+        dlt=dlt_spy,
+        table_name="apollo.artemis.settled_bet",
+        bronze_table_name="apollo.artemis.bronze_settled_bet",
+        keys=["id"],
+        partitions=[],
+        exclude_columns=[],
+    )
+
+    table_properties = dlt_spy.streaming_tables[0]["table_properties"]
+    assert table_properties["delta.enableTypeWidening"] == "true"
+    assert table_properties["delta.autoOptimize.optimizeWrite"] == "true"
+    assert table_properties["delta.autoOptimize.autoCompact"] == "true"
+
+
+def test_silver_streaming_process_merges_extra_table_properties(monkeypatch):
+    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
+    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
+
+    dlt_spy = _DltSpy()
+    main_cdc_utils.silver_streaming_process(
+        dlt=dlt_spy,
+        table_name="apollo.artemis.settled_bet",
+        bronze_table_name="apollo.artemis.bronze_settled_bet",
+        keys=["id"],
+        partitions=[],
+        exclude_columns=[],
+        extra_table_properties={"delta.enableTypeWidening": "false", "my.custom.prop": "42"},
+    )
+
+    table_properties = dlt_spy.streaming_tables[0]["table_properties"]
+    # Caller override wins over the library default.
+    assert table_properties["delta.enableTypeWidening"] == "false"
+    assert table_properties["my.custom.prop"] == "42"
+
+
+def test_create_bronze_table_definition_reads_schema_with_merge_and_enables_type_widening():
+    class _DT:
+        def __init__(self, t):
+            self._t = t
+
+        def simpleString(self):
+            return self._t
+
+    class _Field:
+        def __init__(self, name, type_str):
+            self.name = name
+            self.dataType = _DT(type_str)
+
+    class _DF:
+        def __init__(self, fields):
+            class _Schema:
+                pass
+
+            self.schema = _Schema()
+            self.schema.fields = fields
+
+        def transform(self, _fn):
+            return self
+
+    calls = {"options": {}, "format": None}
+
+    class _Reader:
+        def format(self, fmt):
+            calls["format"] = fmt
+            return self
+
+        def option(self, key, value):
+            calls["options"][key] = value
+            return self
+
+        def load(self, _path):
+            # After a mergeSchema=true read the parquet reader would union
+            # decimal(19,2) and decimal(19,3) footers into decimal(19,3).
+            return _DF([_Field("pnl", "decimal(19,3)"), _Field("id", "bigint")])
+
+    spark = type("_Spark", (), {})()
+    spark.read = _Reader()
+
+    captured = {}
+
+    class _Dlt:
+        def table(self, **kwargs):
+            captured["kwargs"] = kwargs
+
+            def _decorator(fn):
+                captured["fn"] = fn
+                return fn
+
+            return _decorator
+
+    main_cdc_utils.create_bronze_table_definition(
+        spark=spark,
+        dlt=_Dlt(),
+        source_table_name="settled_bet",
+        table_name="apollo.artemis.bronze_settled_bet",
+        files_path="s3://bucket/dms_bronze/artemis/settled_bet/",
+        file_format="parquet",
+        partitions=[],
+        schema_exclude_columns=[],
+        time_diff_history_cdc_timestamp=30,
+        default_value_for_removed_col={},
+        use_managed_file_events=False,
+    )
+
+    # dlt.table just captures the transform — invoke it to exercise the
+    # initial spark.read path (the readStream chain that follows is not
+    # exercised here; the AttributeError it raises is expected).
+    try:
+        captured["fn"]()
+    except AttributeError:
+        pass
+
+    assert calls["format"] == "parquet"
+    assert calls["options"].get("mergeSchema") == "true"
+    assert captured["kwargs"]["table_properties"]["delta.enableTypeWidening"] == "true"

dlt_utils_lib-1.2.2/tests/test_main_cdc_utils.py

@@ -1,99 +0,0 @@
-import dlt_utils.main_cdc_utils as main_cdc_utils
-
-
-class _DltSpy:
-    def __init__(self):
-        self.decorated_tables = []
-        self.streaming_tables = []
-        self.apply_changes_calls = []
-
-    def table(self, **kwargs):
-        self.decorated_tables.append(kwargs)
-
-        def decorator(func):
-            return func
-
-        return decorator
-
-    def create_streaming_table(self, **kwargs):
-        self.streaming_tables.append(kwargs)
-
-    def apply_changes(self, **kwargs):
-        self.apply_changes_calls.append(kwargs)
-
-
-def _build_cdc_table():
-    return {
-        "source_table_name": "customer",
-        "source_schema": "bifrost",
-        "format": "parquet",
-        "keys": ["id"],
-        "exclude_columns": [],
-        "enable_truncate": False,
-    }
-
-
-def test_base_cdc_replication_process_uses_pipeline_schema_configuration(monkeypatch):
-    dlt_spy = _DltSpy()
-    spark = object()
-
-    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
-    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
-
-    main_cdc_utils.base_cdc_replication_process(
-        dlt=dlt_spy,
-        spark=spark,
-        cdc_tables_map=[_build_cdc_table()],
-        bronze_directory="dms_bronze",
-        bucket_name="s3://bucket",
-        bronze_schema="bifrost_bronze",
-        silver_schema="bifrost_silver",
-    )
-
-    assert dlt_spy.decorated_tables[0]["name"] == "bifrost_bronze.customer"
-    assert dlt_spy.streaming_tables[0]["name"] == "bifrost_silver.customer"
-    assert dlt_spy.apply_changes_calls[0]["source"] == "bifrost_bronze.customer"
-    assert dlt_spy.apply_changes_calls[0]["target"] == "bifrost_silver.customer"
-
-
-def test_base_cdc_replication_process_uses_legacy_names_without_both_schemas(monkeypatch):
-    dlt_spy = _DltSpy()
-    spark = object()
-
-    monkeypatch.setattr(main_cdc_utils, "col", lambda value: value)
-    monkeypatch.setattr(main_cdc_utils, "expr", lambda value: value)
-
-    main_cdc_utils.base_cdc_replication_process(
-        dlt=dlt_spy,
-        spark=spark,
-        cdc_tables_map=[_build_cdc_table()],
-        bronze_directory="dms_bronze",
-        bucket_name="s3://bucket",
-        bronze_schema="bifrost_bronze",
-    )
-
-    assert dlt_spy.decorated_tables[0]["name"] == "bronze_customer"
-    assert dlt_spy.streaming_tables[0]["name"] == "customer"
-    assert dlt_spy.apply_changes_calls[0]["source"] == "bronze_customer"
-    assert dlt_spy.apply_changes_calls[0]["target"] == "customer"
-
-
-def test_base_cdc_replication_process_enables_managed_file_events(monkeypatch):
-    captured_bronze_kwargs = {}
-
-    def fake_create_bronze_table_definition(**kwargs):
-        captured_bronze_kwargs.update(kwargs)
-
-    monkeypatch.setattr(main_cdc_utils, "create_bronze_table_definition", fake_create_bronze_table_definition)
-    monkeypatch.setattr(main_cdc_utils, "silver_streaming_process", lambda **kwargs: None)
-
-    main_cdc_utils.base_cdc_replication_process(
-        dlt=object(),
-        spark=object(),
-        cdc_tables_map=[_build_cdc_table()],
-        bronze_directory="dms_bronze",
-        bucket_name="s3://bucket",
-        use_managed_file_events=True,
-    )
-
-    assert captured_bronze_kwargs["use_managed_file_events"] is True