dlt-utils-lib 1.2.6__tar.gz → 1.3.0__tar.gz

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.2.6
+Version: 1.3.0
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.3.0}/dlt_utils/dlt_autoloader_recovery_configuration.py

@@ -2,6 +2,8 @@ import logging
 
 from databricks.sdk import WorkspaceClient
 
+_RECOVERY_DECISION = None
+
 
 def _needs_autoloader_recovery(pipeline_id: str) -> bool:
     event = next(iter(WorkspaceClient().pipelines.list_pipeline_events(
@@ -18,9 +20,20 @@ def _needs_autoloader_recovery(pipeline_id: str) -> bool:
 
 
 def resolve_autoloader_recovery(spark) -> bool:
+    # Cached for the lifetime of the driver process: the decision belongs to the
+    # update as a whole, but callers ask once per table while the pipeline graph
+    # is built. Uncached, a 94-table pipeline makes 94 Pipelines API calls,
+    # serialized on the driver, before a single stream starts.
+    global _RECOVERY_DECISION
+    if _RECOVERY_DECISION is None:
+        _RECOVERY_DECISION = _resolve_autoloader_recovery(spark)
+    return _RECOVERY_DECISION
+
+
+def _resolve_autoloader_recovery(spark) -> bool:
     method = spark.conf.get("autoloader_recovery_method", "AUTO").upper()
     if method == "ON":
         return True
     if method == "AUTO":
         return _needs_autoloader_recovery(spark.conf.get("pipelines.id"))
-    return False
+    return False

dlt_utils_lib-1.3.0/dlt_utils/dlt_table_properties.py

@@ -0,0 +1,32 @@
+import json
+from typing import Optional
+
+# delta.enableTypeWidening lets Delta accept widened column types (e.g. decimal
+# scale bumps) from upstream schema drift without a manual ALTER + full refresh.
+# Applied to every bronze and silver table so future source-side type changes
+# flow through the pipeline dynamically.
+#
+# Shared by the CDC and JSON replication paths rather than defined in each. They
+# held separate copies once and the copies disagreed: the JSON path never gained
+# type widening, so Kafka-sourced tables needed a full refresh for a change the
+# CDC tables absorbed.
+DEFAULT_TABLE_PROPERTIES = {
+    "delta.autoOptimize.optimizeWrite": "true",
+    "delta.autoOptimize.autoCompact": "true",
+    "delta.enableTypeWidening": "true",
+}
+
+
+def build_table_properties(extra: Optional[dict] = None,
+                           column_comments: Optional[dict] = None) -> dict:
+    """Defaults, plus column comments when the caller tracks them, plus overrides.
+
+    `column_comments=None` leaves the key out entirely, which is how bronze
+    tables are built; an empty dict still writes "{}" so the property exists.
+    """
+    props = dict(DEFAULT_TABLE_PROPERTIES)
+    if column_comments is not None:
+        props["column_comments"] = json.dumps(column_comments) if column_comments else "{}"
+    if extra:
+        props.update(extra)
+    return props

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.3.0}/dlt_utils/main_cdc_utils.py

@@ -1,9 +1,9 @@
-import json
 from typing import Optional
 
 from pyspark.sql.functions import col, expr, current_timestamp
 
 from .dlt_autoloader_recovery_configuration import resolve_autoloader_recovery
+from .dlt_table_properties import build_table_properties
 from .dlt_transformations import (
     add_default_value_for_removed_col,
     apply_partitions,
@@ -12,27 +12,6 @@ from .dlt_transformations import (
 )
 
 
-# delta.enableTypeWidening lets Delta accept widened column types (e.g. decimal
-# scale bumps) from upstream schema drift without a manual ALTER + full refresh.
-# Applied to every bronze and silver table so future source-side type changes
-# flow through the pipeline dynamically.
-_DEFAULT_TABLE_PROPERTIES = {
-    "delta.autoOptimize.optimizeWrite": "true",
-    "delta.autoOptimize.autoCompact": "true",
-    "delta.enableTypeWidening": "true",
-}
-
-
-def _build_table_properties(extra: Optional[dict] = None,
-                            column_comments: Optional[dict] = None) -> dict:
-    props = dict(_DEFAULT_TABLE_PROPERTIES)
-    if column_comments is not None:
-        props["column_comments"] = json.dumps(column_comments) if column_comments else "{}"
-    if extra:
-        props.update(extra)
-    return props
-
-
 def _pick_latest_source_file(spark, files_path: str) -> Optional[str]:
     """Return the path of the most recently modified source file, or None.
 
@@ -137,7 +116,7 @@ def create_bronze_table_definition(spark,
                                    use_managed_file_events: bool,
                                    extra_table_properties: Optional[dict] = None,
                                    ):
-    table_properties = _build_table_properties(extra_table_properties)
+    table_properties = build_table_properties(extra_table_properties)
 
     # Resolve the schemaHints outside the @dlt.table body so that the
     # collect() inside _pick_latest_source_file runs once at pipeline
@@ -208,7 +187,7 @@ def silver_streaming_process(dlt,
                              cluster_by_value: list = None,
                              is_cluster_by_auto: bool = True,
                              extra_table_properties: Optional[dict] = None):
-    table_properties = _build_table_properties(
+    table_properties = build_table_properties(
         extra_table_properties,
         column_comments=column_comments,
     )

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.3.0}/dlt_utils/main_json_utils.py

@@ -1,10 +1,10 @@
-import json
 from typing import Callable, Optional
 
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import col, struct
 
 from .dlt_autoloader_recovery_configuration import resolve_autoloader_recovery
+from .dlt_table_properties import build_table_properties
 
 
 def base_json_replication_process(
@@ -29,6 +29,7 @@ def base_json_replication_process(
         is_cluster_by_auto = table.get("is_cluster_by_auto", False)
         keys = table.get("keys")
         sequence_by_columns = table.get("sequence_by_columns")
+        extra_table_properties = table.get("table_properties") or {}
 
         if bronze_schema and silver_schema:
             bronze_table_name = f"{bronze_schema}.{dataset_name}"
@@ -47,6 +48,7 @@ def base_json_replication_process(
             schema_hints=schema_hints,
             partitions=bronze_partitions,
             use_managed_file_events=use_managed_file_events,
+            extra_table_properties=extra_table_properties,
         )
 
         if keys and sequence_by_columns:
@@ -64,6 +66,7 @@ def base_json_replication_process(
                 is_cluster_by_auto=is_cluster_by_auto,
                 keys=keys,
                 sequence_by_columns=sequence_by_columns,
+                extra_table_properties=extra_table_properties,
             )
         else:
             create_silver_json_table_definition(
@@ -78,6 +81,7 @@ def base_json_replication_process(
                 column_comments=column_comments,
                 cluster_by_value=cluster_by_value,
                 is_cluster_by_auto=is_cluster_by_auto,
+                extra_table_properties=extra_table_properties,
             )
 
 
@@ -87,6 +91,7 @@ def _build_autoloader_reader(
         file_format: str,
         schema_hints: Optional[str],
         use_managed_file_events: bool,
+        include_source_file: bool = False,
 ):
     reader = (
         spark.readStream
@@ -108,7 +113,15 @@ def _build_autoloader_reader(
             .option("cloudFiles.validateOptions", "false")
         )
 
-    return reader.load(source_path)
+    loaded = reader.load(source_path)
+
+    if include_source_file:
+        # Which file a row was read from. Off by default because it adds a
+        # column, and a column added to an existing bronze table is a schema
+        # change for everything already reading it.
+        loaded = loaded.withColumn("source_file", col("_metadata.file_path"))
+
+    return loaded
 
 
 def create_bronze_json_table_definition(
@@ -121,11 +134,14 @@ def create_bronze_json_table_definition(
         schema_hints: Optional[str],
         partitions: Optional[list],
         use_managed_file_events: bool,
+        extra_table_properties: Optional[dict] = None,
+        include_source_file: bool = False,
 ):
     table_kwargs = {
         "name": table_name,
         "comment": f"Raw JSON stream data for {dataset_name}.",
         "temporary": False,
+        "table_properties": build_table_properties(extra_table_properties),
     }
 
     if partitions:
@@ -139,6 +155,7 @@ def create_bronze_json_table_definition(
             file_format=file_format,
             schema_hints=schema_hints,
             use_managed_file_events=use_managed_file_events,
+            include_source_file=include_source_file,
         )
 
     return transform_json_to_bronze
@@ -156,17 +173,18 @@ def create_silver_json_table_definition(
         column_comments: Optional[dict] = None,
         cluster_by_value: Optional[list] = None,
         is_cluster_by_auto: bool = False,
+        extra_table_properties: Optional[dict] = None,
 ):
     supports_clustering = not partitions
     table_kwargs = {
         "name": table_name,
         "comment": table_description or f"Structured JSON stream data for {dataset_name}.",
         "temporary": False,
-        "table_properties": {
-            "delta.autoOptimize.optimizeWrite": "true",
-            "delta.autoOptimize.autoCompact": "true",
-            "column_comments": json.dumps(column_comments) if column_comments else "{}",
-        },
+        # `or {}` rather than passing None through: the JSON path has always
+        # written a column_comments property even when it had none, and dropping
+        # the key would change it on every existing silver table. The CDC path
+        # omits it instead, which is why the two calls differ here.
+        "table_properties": build_table_properties(extra_table_properties, column_comments=column_comments or {}),
     }
 
     if partitions:
@@ -200,6 +218,7 @@ def create_silver_json_cdc_definition(
         column_comments: Optional[dict] = None,
         cluster_by_value: Optional[list] = None,
         is_cluster_by_auto: bool = False,
+        extra_table_properties: Optional[dict] = None,
 ):
     staging_view_name = f"{dataset_name}_cdc_source"
     supports_clustering = not partitions
@@ -211,11 +230,8 @@ def create_silver_json_cdc_definition(
 
     dlt.create_streaming_table(
         name=table_name,
-        table_properties={
-            "delta.autoOptimize.optimizeWrite": "true",
-            "delta.autoOptimize.autoCompact": "true",
-            "column_comments": json.dumps(column_comments) if column_comments else "{}",
-        },
+        # See the note in create_silver_json_table_definition on `or {}`.
+        table_properties=build_table_properties(extra_table_properties, column_comments=column_comments or {}),
         comment=table_description or f"Structured JSON stream data for {dataset_name}.",
         partition_cols=partitions if partitions else None,
         cluster_by=cluster_by_value if supports_clustering else None,

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.3.0}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.2.6
+Version: 1.3.0
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.3.0}/dlt_utils_lib.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ setup.py
 dlt_utils/__init__.py
 dlt_utils/dlt_autoloader_recovery_configuration.py
 dlt_utils/dlt_metadata_receiver.py
+dlt_utils/dlt_table_properties.py
 dlt_utils/dlt_transformations.py
 dlt_utils/main_cdc_utils.py
 dlt_utils/main_json_utils.py
@@ -12,6 +13,7 @@ dlt_utils_lib.egg-info/requires.txt
 dlt_utils_lib.egg-info/top_level.txt
 tests/__init__.py
 tests/conftest.py
+tests/test_autoloader_recovery_configuration.py
 tests/test_main_cdc_utils.py
 tests/test_main_json_utils.py
 tests/test_transformation_module.py

dlt_utils_lib-1.3.0/tests/test_autoloader_recovery_configuration.py

@@ -0,0 +1,68 @@
+import pytest
+
+import dlt_utils.dlt_autoloader_recovery_configuration as recovery
+
+
+class _SparkConfStub:
+    def __init__(self, values):
+        self._values = values
+
+    def get(self, key, default=None):
+        return self._values.get(key, default)
+
+
+class _SparkStub:
+    def __init__(self, **values):
+        self.conf = _SparkConfStub(values)
+
+
+@pytest.fixture(autouse=True)
+def reset_cached_decision():
+    recovery._RECOVERY_DECISION = None
+    yield
+    recovery._RECOVERY_DECISION = None
+
+
+def test_method_on_forces_recovery_without_calling_the_api(monkeypatch):
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery", _unexpected_api_call)
+
+    assert recovery.resolve_autoloader_recovery(
+        _SparkStub(autoloader_recovery_method="ON")
+    ) is True
+
+
+def test_method_off_skips_recovery_without_calling_the_api(monkeypatch):
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery", _unexpected_api_call)
+
+    assert recovery.resolve_autoloader_recovery(
+        _SparkStub(autoloader_recovery_method="OFF")
+    ) is False
+
+
+def test_method_auto_asks_the_pipelines_api(monkeypatch):
+    seen = []
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery",
+                        lambda pipeline_id: seen.append(pipeline_id) or True)
+
+    spark = _SparkStub(autoloader_recovery_method="AUTO", **{"pipelines.id": "pipeline-1"})
+
+    assert recovery.resolve_autoloader_recovery(spark) is True
+    assert seen == ["pipeline-1"]
+
+
+def test_decision_is_resolved_once_per_process(monkeypatch):
+    calls = []
+    monkeypatch.setattr(recovery, "_needs_autoloader_recovery",
+                        lambda pipeline_id: calls.append(pipeline_id) or True)
+
+    spark = _SparkStub(autoloader_recovery_method="AUTO", **{"pipelines.id": "pipeline-1"})
+
+    # One call per table while the DLT graph is built; only the first may hit the API.
+    for _ in range(10):
+        assert recovery.resolve_autoloader_recovery(spark) is True
+
+    assert calls == ["pipeline-1"]
+
+
+def _unexpected_api_call(pipeline_id):
+    raise AssertionError(f"Pipelines API should not be called (pipeline_id={pipeline_id})")

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.3.0}/tests/test_main_json_utils.py

@@ -141,3 +141,63 @@ def test_base_json_replication_process_enables_managed_file_events(monkeypatch):
     )
 
     assert captured_bronze_kwargs["use_managed_file_events"] is True
+
+
+def test_bronze_and_silver_enable_type_widening_by_default():
+    dlt_spy = _DltSpy()
+
+    main_json_utils.base_json_replication_process(
+        dlt=dlt_spy,
+        spark=object(),
+        json_tables_map=[_build_json_table()],
+        bronze_schema="bifrost_bronze",
+        silver_schema="bifrost_silver",
+    )
+
+    # Bronze carried no table properties at all before this; a Kafka-sourced
+    # table needed a full refresh for a widening the CDC tables absorbed.
+    for table_kwargs in dlt_spy.decorated_tables:
+        properties = table_kwargs["table_properties"]
+        assert properties["delta.enableTypeWidening"] == "true"
+        assert properties["delta.autoOptimize.optimizeWrite"] == "true"
+        assert properties["delta.autoOptimize.autoCompact"] == "true"
+
+
+def test_cdc_silver_enables_type_widening_by_default(monkeypatch):
+    monkeypatch.setattr(main_json_utils, "col", lambda value: value)
+    monkeypatch.setattr(main_json_utils, "struct", lambda *values: values)
+
+    dlt_spy = _DltSpy()
+    json_table = dict(_build_json_table(), keys=["id"], sequence_by_columns=["kafka_timestamp"])
+
+    main_json_utils.base_json_replication_process(
+        dlt=dlt_spy,
+        spark=object(),
+        json_tables_map=[json_table],
+        bronze_schema="bifrost_bronze",
+        silver_schema="bifrost_silver",
+    )
+
+    assert dlt_spy.streaming_tables[0]["table_properties"]["delta.enableTypeWidening"] == "true"
+
+
+def test_table_properties_override_the_library_defaults():
+    dlt_spy = _DltSpy()
+    json_table = dict(
+        _build_json_table(),
+        table_properties={"delta.enableTypeWidening": "false", "my.custom.prop": "42"},
+    )
+
+    main_json_utils.base_json_replication_process(
+        dlt=dlt_spy,
+        spark=object(),
+        json_tables_map=[json_table],
+        bronze_schema="bifrost_bronze",
+        silver_schema="bifrost_silver",
+    )
+
+    for table_kwargs in dlt_spy.decorated_tables:
+        properties = table_kwargs["table_properties"]
+        # Caller override wins over the library default.
+        assert properties["delta.enableTypeWidening"] == "false"
+        assert properties["my.custom.prop"] == "42"

{dlt_utils_lib-1.2.6 → dlt_utils_lib-1.3.0}/tests/test_transformation_module.py

@@ -7,10 +7,18 @@ from pyspark.sql.functions import expr
 from pyspark.sql.types import TimestampType, StringType
 from datetime import datetime
 from dlt_utils.dlt_transformations import rename_columns, apply_partitions, update_cdc_timestamp, add_default_value_for_removed_col, rename_columns
-# Mock databricks.sdk.runtime if not in a Databricks environment
+# Mock databricks.sdk.runtime if not in a Databricks environment.
+#
+# Catching Exception and not ImportError: importing this module builds a
+# Config and tries to authenticate, so off a cluster it fails in whatever way
+# the surrounding credentials happen to be wrong. In CI that is
+# "more than one authorization method configured: oauth and pat", because the
+# project still carries a PAT next to the OAuth variables. On a laptop it is
+# usually several matching profiles in ~/.databrickscfg. Both mean the same
+# thing here, which is that we are not on a cluster and should mock.
 try:
     from databricks.sdk.runtime import *
-except ImportError:
+except Exception:
     print("Databricks runtime not available. Mocking required modules.")
     mock_dbutils = mock.Mock()
     sys.modules['databricks.sdk.runtime'] = mock.Mock()