dlt-utils-lib 1.3.1__tar.gz → 1.3.2__tar.gz

{dlt_utils_lib-1.3.1 → dlt_utils_lib-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.3.1
+Version: 1.3.2
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.3.1 → dlt_utils_lib-1.3.2}/dlt_utils/dlt_transformations.py

@@ -12,12 +12,20 @@ def apply_partitions(df: DataFrame, partitions: dict):
     return df
 
 
+def add_file_landed_timestamp(df: DataFrame) -> DataFrame:
+    # When the source file landed in storage (its S3 LastModified), so latency can
+    # be measured from there to silver. _metadata exists only on the file source
+    # itself, so apply this straight after load().
+    return df.withColumn('file_landed_timestamp', col('_metadata.file_modification_time'))
+
+
 def update_cdc_timestamp(df: DataFrame, time_diff_threshold: int) -> DataFrame:
     # if cdc_timestamp is null or time difference is greater than threshold, set it to the max timestamp in the row
     timestamp_cols = [col.name for col in df.schema.fields if
                       isinstance(col.dataType, (TimestampType, DateType)) and col.name not in ['cdc_timestamp',
                                                                                                'commit_timestamp',
-                                                                                               'dlt_timestamp']]
+                                                                                               'dlt_timestamp',
+                                                                                               'file_landed_timestamp']]
 
     if timestamp_cols:
         max_timestamp_per_row = None

{dlt_utils_lib-1.3.1 → dlt_utils_lib-1.3.2}/dlt_utils/main_cdc_utils.py

@@ -6,6 +6,7 @@ from .dlt_autoloader_recovery_configuration import resolve_autoloader_recovery
 from .dlt_table_properties import build_table_properties
 from .dlt_transformations import (
     add_default_value_for_removed_col,
+    add_file_landed_timestamp,
     apply_partitions,
     rename_columns,
     update_cdc_timestamp,
@@ -171,6 +172,7 @@ def create_bronze_table_definition(spark,
             )
 
         return reader.load(files_path) \
+            .transform(add_file_landed_timestamp) \
             .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp')) \
             .withColumn('ar_h_change_seq', col('ar_h_change_seq').cast('string')) \
             .withColumn('dlt_timestamp', current_timestamp()) \

{dlt_utils_lib-1.3.1 → dlt_utils_lib-1.3.2}/dlt_utils/main_json_utils.py

@@ -5,6 +5,7 @@ from pyspark.sql.functions import col, struct
 
 from .dlt_autoloader_recovery_configuration import resolve_autoloader_recovery
 from .dlt_table_properties import build_table_properties
+from .dlt_transformations import add_file_landed_timestamp
 
 
 def base_json_replication_process(
@@ -113,7 +114,7 @@ def _build_autoloader_reader(
             .option("cloudFiles.validateOptions", "false")
         )
 
-    loaded = reader.load(source_path)
+    loaded = reader.load(source_path).transform(add_file_landed_timestamp)
 
     if include_source_file:
         # Which file a row was read from. Off by default because it adds a

{dlt_utils_lib-1.3.1 → dlt_utils_lib-1.3.2}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.3.1
+Version: 1.3.2
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.3.1 → dlt_utils_lib-1.3.2}/tests/test_main_cdc_utils.py

@@ -1,6 +1,9 @@
+from unittest import mock
+
 import pytest
 
 import dlt_utils.main_cdc_utils as main_cdc_utils
+from dlt_utils.dlt_transformations import add_file_landed_timestamp
 
 
 class _DltSpy:
@@ -263,6 +266,63 @@ def test_create_bronze_table_definition_falls_back_to_directory_when_listing_emp
     assert load_paths == [directory]
 
 
+class _FakeStream:
+    """Records what the bronze body does to the loaded stream, in order."""
+
+    def __init__(self):
+        self.steps = []
+
+    def transform(self, fn):
+        self.steps.append(fn)
+        return self
+
+    def withColumn(self, name, _value):
+        self.steps.append(name)
+        return self
+
+
+class _FakeStreamReader:
+    def __init__(self, stream):
+        self.stream = stream
+
+    def format(self, _fmt):
+        return self
+
+    def option(self, _key, _value):
+        return self
+
+    def load(self, _path):
+        return self.stream
+
+
+def test_bronze_takes_file_landed_timestamp_straight_from_the_source(monkeypatch):
+    stream = _FakeStream()
+    spark = _make_spark_with_reader([], {})
+    spark.readStream = _FakeStreamReader(stream)
+    monkeypatch.setattr(main_cdc_utils, "_pick_latest_source_file", lambda *_: None)
+    monkeypatch.setattr(main_cdc_utils, "resolve_autoloader_recovery", lambda _spark: False)
+    monkeypatch.setattr(main_cdc_utils, "col", lambda _name: mock.MagicMock())
+    monkeypatch.setattr(main_cdc_utils, "current_timestamp", lambda: None)
+
+    bronze = main_cdc_utils.create_bronze_table_definition(
+        spark=spark,
+        dlt=_DltSpy(),
+        source_table_name="settled_bet",
+        table_name="artemis_bronze.settled_bet",
+        files_path="/Volumes/artemis_prod/stage/dms_raw/settled_bet/",
+        file_format="parquet",
+        partitions=[],
+        schema_exclude_columns=[],
+        time_diff_history_cdc_timestamp=30,
+        default_value_for_removed_col={},
+        use_managed_file_events=True,
+    )
+    bronze()
+
+    # _metadata resolves only on the file source, so nothing may come before it.
+    assert stream.steps[0] is add_file_landed_timestamp
+
+
 @pytest.mark.parametrize("source_options, expected", [
     ({"bucket_name": "s3://bucket", "bronze_directory": "dms_bronze"},
      "s3://bucket/dms_bronze/bifrost/customer/"),

{dlt_utils_lib-1.3.1 → dlt_utils_lib-1.3.2}/tests/test_main_json_utils.py

@@ -1,4 +1,5 @@
 import dlt_utils.main_json_utils as main_json_utils
+from dlt_utils.dlt_transformations import add_file_landed_timestamp
 
 
 class _DltSpy:
@@ -201,3 +202,48 @@ def test_table_properties_override_the_library_defaults():
         # Caller override wins over the library default.
         assert properties["delta.enableTypeWidening"] == "false"
         assert properties["my.custom.prop"] == "42"
+
+
+class _FakeStream:
+    def __init__(self):
+        self.steps = []
+
+    def transform(self, fn):
+        self.steps.append(fn)
+        return self
+
+    def withColumn(self, name, _value):
+        self.steps.append(name)
+        return self
+
+
+class _FakeSpark:
+    def __init__(self, stream):
+        self.readStream = self
+        self._stream = stream
+
+    def format(self, _fmt):
+        return self
+
+    def option(self, _key, _value):
+        return self
+
+    def load(self, _path):
+        return self._stream
+
+
+def test_bronze_takes_file_landed_timestamp_straight_from_the_source(monkeypatch):
+    monkeypatch.setattr(main_json_utils, "resolve_autoloader_recovery", lambda _spark: False)
+    monkeypatch.setattr(main_json_utils, "col", lambda value: value)
+    stream = _FakeStream()
+
+    main_json_utils._build_autoloader_reader(
+        spark=_FakeSpark(stream),
+        source_path="s3://bucket/raw/exposure_state",
+        file_format="json",
+        schema_hints=None,
+        use_managed_file_events=True,
+        include_source_file=True,
+    )
+
+    assert stream.steps == [add_file_landed_timestamp, "source_file"]

{dlt_utils_lib-1.3.1 → dlt_utils_lib-1.3.2}/tests/test_transformation_module.py

@@ -6,7 +6,7 @@ from pyspark.sql import SparkSession, Row
 from pyspark.sql.functions import expr
 from pyspark.sql.types import TimestampType, StringType
 from datetime import datetime
-from dlt_utils.dlt_transformations import rename_columns, apply_partitions, update_cdc_timestamp, add_default_value_for_removed_col, rename_columns
+from dlt_utils.dlt_transformations import rename_columns, apply_partitions, update_cdc_timestamp, add_default_value_for_removed_col, rename_columns, add_file_landed_timestamp
 # Mock databricks.sdk.runtime if not in a Databricks environment.
 #
 # Catching Exception and not ImportError: importing this module builds a
@@ -79,6 +79,29 @@ def test_update_cdc_timestamp(spark):
     assert result[1]['cdc_timestamp'] == datetime(2024, 1, 1)
 
 
+def test_update_cdc_timestamp_ignores_file_landed_timestamp(spark):
+    data = [
+        Row(id=1, updated_at=datetime(2024, 1, 10), file_landed_timestamp=datetime(2024, 3, 1), cdc_timestamp=None),
+        Row(id=2, updated_at=datetime(2024, 1, 10), file_landed_timestamp=datetime(2024, 3, 1),
+            cdc_timestamp=datetime(2024, 1, 9)),
+    ]
+
+    result = update_cdc_timestamp(spark.createDataFrame(data), time_diff_threshold=5).collect()
+
+    assert result[0]['cdc_timestamp'] == datetime(2024, 1, 10)
+
+
+def test_add_file_landed_timestamp_is_the_file_modification_time(spark, tmp_path):
+    path = str(tmp_path / "landed")
+    spark.createDataFrame([Row(id=1)]).coalesce(1).write.parquet(path)
+    part = next(name for name in os.listdir(path) if name.endswith(".parquet"))
+    modified = datetime.fromtimestamp(int(os.path.getmtime(os.path.join(path, part))))
+
+    result = add_file_landed_timestamp(spark.read.parquet(path)).collect()[0]
+
+    assert result['file_landed_timestamp'].replace(microsecond=0) == modified
+
+
 
 
 def test_add_default_value_for_removed_col(spark):