dlt-utils-lib 1.2.7__tar.gz → 1.3.1__tar.gz

{dlt_utils_lib-1.2.7 → dlt_utils_lib-1.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.2.7
+Version: 1.3.1
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

dlt_utils_lib-1.3.1/dlt_utils/dlt_table_properties.py

@@ -0,0 +1,32 @@
+import json
+from typing import Optional
+
+# delta.enableTypeWidening lets Delta accept widened column types (e.g. decimal
+# scale bumps) from upstream schema drift without a manual ALTER + full refresh.
+# Applied to every bronze and silver table so future source-side type changes
+# flow through the pipeline dynamically.
+#
+# Shared by the CDC and JSON replication paths rather than defined in each. They
+# held separate copies once and the copies disagreed: the JSON path never gained
+# type widening, so Kafka-sourced tables needed a full refresh for a change the
+# CDC tables absorbed.
+DEFAULT_TABLE_PROPERTIES = {
+    "delta.autoOptimize.optimizeWrite": "true",
+    "delta.autoOptimize.autoCompact": "true",
+    "delta.enableTypeWidening": "true",
+}
+
+
+def build_table_properties(extra: Optional[dict] = None,
+                           column_comments: Optional[dict] = None) -> dict:
+    """Defaults, plus column comments when the caller tracks them, plus overrides.
+
+    `column_comments=None` leaves the key out entirely, which is how bronze
+    tables are built; an empty dict still writes "{}" so the property exists.
+    """
+    props = dict(DEFAULT_TABLE_PROPERTIES)
+    if column_comments is not None:
+        props["column_comments"] = json.dumps(column_comments) if column_comments else "{}"
+    if extra:
+        props.update(extra)
+    return props

{dlt_utils_lib-1.2.7 → dlt_utils_lib-1.3.1}/dlt_utils/main_cdc_utils.py

@@ -1,9 +1,9 @@
-import json
 from typing import Optional
 
 from pyspark.sql.functions import col, expr, current_timestamp
 
 from .dlt_autoloader_recovery_configuration import resolve_autoloader_recovery
+from .dlt_table_properties import build_table_properties
 from .dlt_transformations import (
     add_default_value_for_removed_col,
     apply_partitions,
@@ -12,27 +12,6 @@ from .dlt_transformations import (
 )
 
 
-# delta.enableTypeWidening lets Delta accept widened column types (e.g. decimal
-# scale bumps) from upstream schema drift without a manual ALTER + full refresh.
-# Applied to every bronze and silver table so future source-side type changes
-# flow through the pipeline dynamically.
-_DEFAULT_TABLE_PROPERTIES = {
-    "delta.autoOptimize.optimizeWrite": "true",
-    "delta.autoOptimize.autoCompact": "true",
-    "delta.enableTypeWidening": "true",
-}
-
-
-def _build_table_properties(extra: Optional[dict] = None,
-                            column_comments: Optional[dict] = None) -> dict:
-    props = dict(_DEFAULT_TABLE_PROPERTIES)
-    if column_comments is not None:
-        props["column_comments"] = json.dumps(column_comments) if column_comments else "{}"
-    if extra:
-        props.update(extra)
-    return props
-
-
 def _pick_latest_source_file(spark, files_path: str) -> Optional[str]:
     """Return the path of the most recently modified source file, or None.
 
@@ -60,22 +39,29 @@ def _pick_latest_source_file(spark, files_path: str) -> Optional[str]:
 def base_cdc_replication_process(dlt,
                                  spark,
                                  cdc_tables_map: list,
-                                 bronze_directory: str,
-                                 bucket_name: str,
+                                 bronze_directory: Optional[str] = None,
+                                 bucket_name: Optional[str] = None,
                                  time_diff_history_cdc_timestamp: int = 30,
                                  bronze_schema: Optional[str] = None,
                                  silver_schema: Optional[str] = None,
-                                 use_managed_file_events: bool = False):
+                                 use_managed_file_events: bool = False,
+                                 volume_source_path: Optional[str] = None):
+    if volume_source_path is None and (not bronze_directory or not bucket_name):
+        raise ValueError("Provide volume_source_path or both bucket_name and bronze_directory")
+
     # parse list with tables for replication
     for table in cdc_tables_map:
         source_table_name = table['source_table_name']
-        source_schema = table['source_schema']
         file_format = table['format']
         keys = table['keys']
         exclude_columns = table['exclude_columns']
         enable_truncate = table['enable_truncate']
-        s3_files_path = f"{source_schema}/{source_table_name}/"
-        bronze_table_path = f"{bucket_name}/{bronze_directory}/{s3_files_path}"
+        # The volume path already includes the source schema.
+        if volume_source_path is not None:
+            bronze_table_path = f"{volume_source_path.rstrip('/')}/{source_table_name}/"
+        else:
+            source_schema = table['source_schema']
+            bronze_table_path = f"{bucket_name}/{bronze_directory}/{source_schema}/{source_table_name}/"
 
         # Optional parameters
         default_value_for_removed_col = table.get('default_value_for_removed_col', {})
@@ -137,7 +123,7 @@ def create_bronze_table_definition(spark,
                                    use_managed_file_events: bool,
                                    extra_table_properties: Optional[dict] = None,
                                    ):
-    table_properties = _build_table_properties(extra_table_properties)
+    table_properties = build_table_properties(extra_table_properties)
 
     # Resolve the schemaHints outside the @dlt.table body so that the
     # collect() inside _pick_latest_source_file runs once at pipeline
@@ -208,7 +194,7 @@ def silver_streaming_process(dlt,
                              cluster_by_value: list = None,
                              is_cluster_by_auto: bool = True,
                              extra_table_properties: Optional[dict] = None):
-    table_properties = _build_table_properties(
+    table_properties = build_table_properties(
         extra_table_properties,
         column_comments=column_comments,
     )

{dlt_utils_lib-1.2.7 → dlt_utils_lib-1.3.1}/dlt_utils/main_json_utils.py

@@ -1,10 +1,10 @@
-import json
 from typing import Callable, Optional
 
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import col, struct
 
 from .dlt_autoloader_recovery_configuration import resolve_autoloader_recovery
+from .dlt_table_properties import build_table_properties
 
 
 def base_json_replication_process(
@@ -29,6 +29,7 @@ def base_json_replication_process(
         is_cluster_by_auto = table.get("is_cluster_by_auto", False)
         keys = table.get("keys")
         sequence_by_columns = table.get("sequence_by_columns")
+        extra_table_properties = table.get("table_properties") or {}
 
         if bronze_schema and silver_schema:
             bronze_table_name = f"{bronze_schema}.{dataset_name}"
@@ -47,6 +48,7 @@ def base_json_replication_process(
             schema_hints=schema_hints,
             partitions=bronze_partitions,
             use_managed_file_events=use_managed_file_events,
+            extra_table_properties=extra_table_properties,
         )
 
         if keys and sequence_by_columns:
@@ -64,6 +66,7 @@ def base_json_replication_process(
                 is_cluster_by_auto=is_cluster_by_auto,
                 keys=keys,
                 sequence_by_columns=sequence_by_columns,
+                extra_table_properties=extra_table_properties,
             )
         else:
             create_silver_json_table_definition(
@@ -78,6 +81,7 @@ def base_json_replication_process(
                 column_comments=column_comments,
                 cluster_by_value=cluster_by_value,
                 is_cluster_by_auto=is_cluster_by_auto,
+                extra_table_properties=extra_table_properties,
             )
 
 
@@ -87,6 +91,7 @@ def _build_autoloader_reader(
         file_format: str,
         schema_hints: Optional[str],
         use_managed_file_events: bool,
+        include_source_file: bool = False,
 ):
     reader = (
         spark.readStream
@@ -108,7 +113,15 @@ def _build_autoloader_reader(
             .option("cloudFiles.validateOptions", "false")
         )
 
-    return reader.load(source_path)
+    loaded = reader.load(source_path)
+
+    if include_source_file:
+        # Which file a row was read from. Off by default because it adds a
+        # column, and a column added to an existing bronze table is a schema
+        # change for everything already reading it.
+        loaded = loaded.withColumn("source_file", col("_metadata.file_path"))
+
+    return loaded
 
 
 def create_bronze_json_table_definition(
@@ -121,11 +134,14 @@ def create_bronze_json_table_definition(
         schema_hints: Optional[str],
         partitions: Optional[list],
         use_managed_file_events: bool,
+        extra_table_properties: Optional[dict] = None,
+        include_source_file: bool = False,
 ):
     table_kwargs = {
         "name": table_name,
         "comment": f"Raw JSON stream data for {dataset_name}.",
         "temporary": False,
+        "table_properties": build_table_properties(extra_table_properties),
     }
 
     if partitions:
@@ -139,6 +155,7 @@ def create_bronze_json_table_definition(
             file_format=file_format,
             schema_hints=schema_hints,
             use_managed_file_events=use_managed_file_events,
+            include_source_file=include_source_file,
         )
 
     return transform_json_to_bronze
@@ -156,17 +173,18 @@ def create_silver_json_table_definition(
         column_comments: Optional[dict] = None,
         cluster_by_value: Optional[list] = None,
         is_cluster_by_auto: bool = False,
+        extra_table_properties: Optional[dict] = None,
 ):
     supports_clustering = not partitions
     table_kwargs = {
         "name": table_name,
         "comment": table_description or f"Structured JSON stream data for {dataset_name}.",
         "temporary": False,
-        "table_properties": {
-            "delta.autoOptimize.optimizeWrite": "true",
-            "delta.autoOptimize.autoCompact": "true",
-            "column_comments": json.dumps(column_comments) if column_comments else "{}",
-        },
+        # `or {}` rather than passing None through: the JSON path has always
+        # written a column_comments property even when it had none, and dropping
+        # the key would change it on every existing silver table. The CDC path
+        # omits it instead, which is why the two calls differ here.
+        "table_properties": build_table_properties(extra_table_properties, column_comments=column_comments or {}),
     }
 
     if partitions:
@@ -200,6 +218,7 @@ def create_silver_json_cdc_definition(
         column_comments: Optional[dict] = None,
         cluster_by_value: Optional[list] = None,
         is_cluster_by_auto: bool = False,
+        extra_table_properties: Optional[dict] = None,
 ):
     staging_view_name = f"{dataset_name}_cdc_source"
     supports_clustering = not partitions
@@ -211,11 +230,8 @@ def create_silver_json_cdc_definition(
 
     dlt.create_streaming_table(
         name=table_name,
-        table_properties={
-            "delta.autoOptimize.optimizeWrite": "true",
-            "delta.autoOptimize.autoCompact": "true",
-            "column_comments": json.dumps(column_comments) if column_comments else "{}",
-        },
+        # See the note in create_silver_json_table_definition on `or {}`.
+        table_properties=build_table_properties(extra_table_properties, column_comments=column_comments or {}),
         comment=table_description or f"Structured JSON stream data for {dataset_name}.",
         partition_cols=partitions if partitions else None,
         cluster_by=cluster_by_value if supports_clustering else None,

{dlt_utils_lib-1.2.7 → dlt_utils_lib-1.3.1}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.2.7
+Version: 1.3.1
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.7 → dlt_utils_lib-1.3.1}/dlt_utils_lib.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ setup.py
 dlt_utils/__init__.py
 dlt_utils/dlt_autoloader_recovery_configuration.py
 dlt_utils/dlt_metadata_receiver.py
+dlt_utils/dlt_table_properties.py
 dlt_utils/dlt_transformations.py
 dlt_utils/main_cdc_utils.py
 dlt_utils/main_json_utils.py

{dlt_utils_lib-1.2.7 → dlt_utils_lib-1.3.1}/tests/test_main_cdc_utils.py

@@ -1,3 +1,5 @@
+import pytest
+
 import dlt_utils.main_cdc_utils as main_cdc_utils
 
 
@@ -259,3 +261,49 @@ def test_create_bronze_table_definition_falls_back_to_directory_when_listing_emp
     )
 
     assert load_paths == [directory]
+
+
+@pytest.mark.parametrize("source_options, expected", [
+    ({"bucket_name": "s3://bucket", "bronze_directory": "dms_bronze"},
+     "s3://bucket/dms_bronze/bifrost/customer/"),
+    ({"volume_source_path": "/Volumes/artemis_prod/stage/dms_raw/"},
+     "/Volumes/artemis_prod/stage/dms_raw/customer/"),
+    ({"volume_source_path": "/Volumes/artemis_prod/stage/dms_raw",
+      "bucket_name": "s3://bucket", "bronze_directory": "dms_bronze"},
+     "/Volumes/artemis_prod/stage/dms_raw/customer/"),
+])
+def test_base_cdc_source_paths(monkeypatch, source_options, expected):
+    captured = []
+    monkeypatch.setattr(main_cdc_utils, "create_bronze_table_definition", lambda **kw: captured.append(kw))
+    monkeypatch.setattr(main_cdc_utils, "silver_streaming_process", lambda **kw: None)
+    table = _build_cdc_table()
+    if "volume_source_path" in source_options:
+        del table["source_schema"]
+    main_cdc_utils.base_cdc_replication_process(
+        dlt=object(), spark=object(), cdc_tables_map=[table],
+        **source_options,
+    )
+    assert captured[0]["files_path"] == expected
+
+
+@pytest.mark.parametrize("source_options", [
+    {},
+    {"bucket_name": "s3://bucket"},
+    {"bronze_directory": "dms_bronze"},
+])
+def test_base_cdc_requires_complete_source_configuration(source_options):
+    with pytest.raises(ValueError, match="Provide volume_source_path or both bucket_name and bronze_directory"):
+        main_cdc_utils.base_cdc_replication_process(
+            dlt=object(), spark=object(), cdc_tables_map=[_build_cdc_table()],
+            **source_options,
+        )
+
+
+def test_base_cdc_s3_requires_source_schema():
+    table = _build_cdc_table()
+    del table["source_schema"]
+    with pytest.raises(KeyError, match="source_schema"):
+        main_cdc_utils.base_cdc_replication_process(
+            dlt=object(), spark=object(), cdc_tables_map=[table],
+            bucket_name="s3://bucket", bronze_directory="dms_bronze",
+        )

{dlt_utils_lib-1.2.7 → dlt_utils_lib-1.3.1}/tests/test_main_json_utils.py

@@ -141,3 +141,63 @@ def test_base_json_replication_process_enables_managed_file_events(monkeypatch):
     )
 
     assert captured_bronze_kwargs["use_managed_file_events"] is True
+
+
+def test_bronze_and_silver_enable_type_widening_by_default():
+    dlt_spy = _DltSpy()
+
+    main_json_utils.base_json_replication_process(
+        dlt=dlt_spy,
+        spark=object(),
+        json_tables_map=[_build_json_table()],
+        bronze_schema="bifrost_bronze",
+        silver_schema="bifrost_silver",
+    )
+
+    # Bronze carried no table properties at all before this; a Kafka-sourced
+    # table needed a full refresh for a widening the CDC tables absorbed.
+    for table_kwargs in dlt_spy.decorated_tables:
+        properties = table_kwargs["table_properties"]
+        assert properties["delta.enableTypeWidening"] == "true"
+        assert properties["delta.autoOptimize.optimizeWrite"] == "true"
+        assert properties["delta.autoOptimize.autoCompact"] == "true"
+
+
+def test_cdc_silver_enables_type_widening_by_default(monkeypatch):
+    monkeypatch.setattr(main_json_utils, "col", lambda value: value)
+    monkeypatch.setattr(main_json_utils, "struct", lambda *values: values)
+
+    dlt_spy = _DltSpy()
+    json_table = dict(_build_json_table(), keys=["id"], sequence_by_columns=["kafka_timestamp"])
+
+    main_json_utils.base_json_replication_process(
+        dlt=dlt_spy,
+        spark=object(),
+        json_tables_map=[json_table],
+        bronze_schema="bifrost_bronze",
+        silver_schema="bifrost_silver",
+    )
+
+    assert dlt_spy.streaming_tables[0]["table_properties"]["delta.enableTypeWidening"] == "true"
+
+
+def test_table_properties_override_the_library_defaults():
+    dlt_spy = _DltSpy()
+    json_table = dict(
+        _build_json_table(),
+        table_properties={"delta.enableTypeWidening": "false", "my.custom.prop": "42"},
+    )
+
+    main_json_utils.base_json_replication_process(
+        dlt=dlt_spy,
+        spark=object(),
+        json_tables_map=[json_table],
+        bronze_schema="bifrost_bronze",
+        silver_schema="bifrost_silver",
+    )
+
+    for table_kwargs in dlt_spy.decorated_tables:
+        properties = table_kwargs["table_properties"]
+        # Caller override wins over the library default.
+        assert properties["delta.enableTypeWidening"] == "false"
+        assert properties["my.custom.prop"] == "42"

{dlt_utils_lib-1.2.7 → dlt_utils_lib-1.3.1}/tests/test_transformation_module.py

@@ -7,10 +7,18 @@ from pyspark.sql.functions import expr
 from pyspark.sql.types import TimestampType, StringType
 from datetime import datetime
 from dlt_utils.dlt_transformations import rename_columns, apply_partitions, update_cdc_timestamp, add_default_value_for_removed_col, rename_columns
-# Mock databricks.sdk.runtime if not in a Databricks environment
+# Mock databricks.sdk.runtime if not in a Databricks environment.
+#
+# Catching Exception and not ImportError: importing this module builds a
+# Config and tries to authenticate, so off a cluster it fails in whatever way
+# the surrounding credentials happen to be wrong. In CI that is
+# "more than one authorization method configured: oauth and pat", because the
+# project still carries a PAT next to the OAuth variables. On a laptop it is
+# usually several matching profiles in ~/.databrickscfg. Both mean the same
+# thing here, which is that we are not on a cluster and should mock.
 try:
     from databricks.sdk.runtime import *
-except ImportError:
+except Exception:
     print("Databricks runtime not available. Mocking required modules.")
     mock_dbutils = mock.Mock()
     sys.modules['databricks.sdk.runtime'] = mock.Mock()