dlt-utils-lib 1.3.0__tar.gz → 1.3.2__tar.gz

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.3.0
+Version: 1.3.2
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.2}/dlt_utils/dlt_transformations.py

@@ -12,12 +12,20 @@ def apply_partitions(df: DataFrame, partitions: dict):
     return df
 
 
+def add_file_landed_timestamp(df: DataFrame) -> DataFrame:
+    # When the source file landed in storage (its S3 LastModified), so latency can
+    # be measured from there to silver. _metadata exists only on the file source
+    # itself, so apply this straight after load().
+    return df.withColumn('file_landed_timestamp', col('_metadata.file_modification_time'))
+
+
 def update_cdc_timestamp(df: DataFrame, time_diff_threshold: int) -> DataFrame:
     # if cdc_timestamp is null or time difference is greater than threshold, set it to the max timestamp in the row
     timestamp_cols = [col.name for col in df.schema.fields if
                       isinstance(col.dataType, (TimestampType, DateType)) and col.name not in ['cdc_timestamp',
                                                                                                'commit_timestamp',
-                                                                                               'dlt_timestamp']]
+                                                                                               'dlt_timestamp',
+                                                                                               'file_landed_timestamp']]
 
     if timestamp_cols:
         max_timestamp_per_row = None

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.2}/dlt_utils/main_cdc_utils.py

@@ -6,6 +6,7 @@ from .dlt_autoloader_recovery_configuration import resolve_autoloader_recovery
 from .dlt_table_properties import build_table_properties
 from .dlt_transformations import (
     add_default_value_for_removed_col,
+    add_file_landed_timestamp,
     apply_partitions,
     rename_columns,
     update_cdc_timestamp,
@@ -39,22 +40,29 @@ def _pick_latest_source_file(spark, files_path: str) -> Optional[str]:
 def base_cdc_replication_process(dlt,
                                  spark,
                                  cdc_tables_map: list,
-                                 bronze_directory: str,
-                                 bucket_name: str,
+                                 bronze_directory: Optional[str] = None,
+                                 bucket_name: Optional[str] = None,
                                  time_diff_history_cdc_timestamp: int = 30,
                                  bronze_schema: Optional[str] = None,
                                  silver_schema: Optional[str] = None,
-                                 use_managed_file_events: bool = False):
+                                 use_managed_file_events: bool = False,
+                                 volume_source_path: Optional[str] = None):
+    if volume_source_path is None and (not bronze_directory or not bucket_name):
+        raise ValueError("Provide volume_source_path or both bucket_name and bronze_directory")
+
     # parse list with tables for replication
     for table in cdc_tables_map:
         source_table_name = table['source_table_name']
-        source_schema = table['source_schema']
         file_format = table['format']
         keys = table['keys']
         exclude_columns = table['exclude_columns']
         enable_truncate = table['enable_truncate']
-        s3_files_path = f"{source_schema}/{source_table_name}/"
-        bronze_table_path = f"{bucket_name}/{bronze_directory}/{s3_files_path}"
+        # The volume path already includes the source schema.
+        if volume_source_path is not None:
+            bronze_table_path = f"{volume_source_path.rstrip('/')}/{source_table_name}/"
+        else:
+            source_schema = table['source_schema']
+            bronze_table_path = f"{bucket_name}/{bronze_directory}/{source_schema}/{source_table_name}/"
 
         # Optional parameters
         default_value_for_removed_col = table.get('default_value_for_removed_col', {})
@@ -164,6 +172,7 @@ def create_bronze_table_definition(spark,
             )
 
         return reader.load(files_path) \
+            .transform(add_file_landed_timestamp) \
             .withColumn('cdc_timestamp', col('cdc_timestamp').cast('timestamp')) \
             .withColumn('ar_h_change_seq', col('ar_h_change_seq').cast('string')) \
             .withColumn('dlt_timestamp', current_timestamp()) \

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.2}/dlt_utils/main_json_utils.py

@@ -5,6 +5,7 @@ from pyspark.sql.functions import col, struct
 
 from .dlt_autoloader_recovery_configuration import resolve_autoloader_recovery
 from .dlt_table_properties import build_table_properties
+from .dlt_transformations import add_file_landed_timestamp
 
 
 def base_json_replication_process(
@@ -113,7 +114,7 @@ def _build_autoloader_reader(
             .option("cloudFiles.validateOptions", "false")
         )
 
-    loaded = reader.load(source_path)
+    loaded = reader.load(source_path).transform(add_file_landed_timestamp)
 
     if include_source_file:
         # Which file a row was read from. Off by default because it adds a

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.2}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.3.0
+Version: 1.3.2
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.2}/tests/test_main_cdc_utils.py

@@ -1,4 +1,9 @@
+from unittest import mock
+
+import pytest
+
 import dlt_utils.main_cdc_utils as main_cdc_utils
+from dlt_utils.dlt_transformations import add_file_landed_timestamp
 
 
 class _DltSpy:
@@ -259,3 +264,106 @@ def test_create_bronze_table_definition_falls_back_to_directory_when_listing_emp
     )
 
     assert load_paths == [directory]
+
+
+class _FakeStream:
+    """Records what the bronze body does to the loaded stream, in order."""
+
+    def __init__(self):
+        self.steps = []
+
+    def transform(self, fn):
+        self.steps.append(fn)
+        return self
+
+    def withColumn(self, name, _value):
+        self.steps.append(name)
+        return self
+
+
+class _FakeStreamReader:
+    def __init__(self, stream):
+        self.stream = stream
+
+    def format(self, _fmt):
+        return self
+
+    def option(self, _key, _value):
+        return self
+
+    def load(self, _path):
+        return self.stream
+
+
+def test_bronze_takes_file_landed_timestamp_straight_from_the_source(monkeypatch):
+    stream = _FakeStream()
+    spark = _make_spark_with_reader([], {})
+    spark.readStream = _FakeStreamReader(stream)
+    monkeypatch.setattr(main_cdc_utils, "_pick_latest_source_file", lambda *_: None)
+    monkeypatch.setattr(main_cdc_utils, "resolve_autoloader_recovery", lambda _spark: False)
+    monkeypatch.setattr(main_cdc_utils, "col", lambda _name: mock.MagicMock())
+    monkeypatch.setattr(main_cdc_utils, "current_timestamp", lambda: None)
+
+    bronze = main_cdc_utils.create_bronze_table_definition(
+        spark=spark,
+        dlt=_DltSpy(),
+        source_table_name="settled_bet",
+        table_name="artemis_bronze.settled_bet",
+        files_path="/Volumes/artemis_prod/stage/dms_raw/settled_bet/",
+        file_format="parquet",
+        partitions=[],
+        schema_exclude_columns=[],
+        time_diff_history_cdc_timestamp=30,
+        default_value_for_removed_col={},
+        use_managed_file_events=True,
+    )
+    bronze()
+
+    # _metadata resolves only on the file source, so nothing may come before it.
+    assert stream.steps[0] is add_file_landed_timestamp
+
+
+@pytest.mark.parametrize("source_options, expected", [
+    ({"bucket_name": "s3://bucket", "bronze_directory": "dms_bronze"},
+     "s3://bucket/dms_bronze/bifrost/customer/"),
+    ({"volume_source_path": "/Volumes/artemis_prod/stage/dms_raw/"},
+     "/Volumes/artemis_prod/stage/dms_raw/customer/"),
+    ({"volume_source_path": "/Volumes/artemis_prod/stage/dms_raw",
+      "bucket_name": "s3://bucket", "bronze_directory": "dms_bronze"},
+     "/Volumes/artemis_prod/stage/dms_raw/customer/"),
+])
+def test_base_cdc_source_paths(monkeypatch, source_options, expected):
+    captured = []
+    monkeypatch.setattr(main_cdc_utils, "create_bronze_table_definition", lambda **kw: captured.append(kw))
+    monkeypatch.setattr(main_cdc_utils, "silver_streaming_process", lambda **kw: None)
+    table = _build_cdc_table()
+    if "volume_source_path" in source_options:
+        del table["source_schema"]
+    main_cdc_utils.base_cdc_replication_process(
+        dlt=object(), spark=object(), cdc_tables_map=[table],
+        **source_options,
+    )
+    assert captured[0]["files_path"] == expected
+
+
+@pytest.mark.parametrize("source_options", [
+    {},
+    {"bucket_name": "s3://bucket"},
+    {"bronze_directory": "dms_bronze"},
+])
+def test_base_cdc_requires_complete_source_configuration(source_options):
+    with pytest.raises(ValueError, match="Provide volume_source_path or both bucket_name and bronze_directory"):
+        main_cdc_utils.base_cdc_replication_process(
+            dlt=object(), spark=object(), cdc_tables_map=[_build_cdc_table()],
+            **source_options,
+        )
+
+
+def test_base_cdc_s3_requires_source_schema():
+    table = _build_cdc_table()
+    del table["source_schema"]
+    with pytest.raises(KeyError, match="source_schema"):
+        main_cdc_utils.base_cdc_replication_process(
+            dlt=object(), spark=object(), cdc_tables_map=[table],
+            bucket_name="s3://bucket", bronze_directory="dms_bronze",
+        )

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.2}/tests/test_main_json_utils.py

@@ -1,4 +1,5 @@
 import dlt_utils.main_json_utils as main_json_utils
+from dlt_utils.dlt_transformations import add_file_landed_timestamp
 
 
 class _DltSpy:
@@ -201,3 +202,48 @@ def test_table_properties_override_the_library_defaults():
         # Caller override wins over the library default.
         assert properties["delta.enableTypeWidening"] == "false"
         assert properties["my.custom.prop"] == "42"
+
+
+class _FakeStream:
+    def __init__(self):
+        self.steps = []
+
+    def transform(self, fn):
+        self.steps.append(fn)
+        return self
+
+    def withColumn(self, name, _value):
+        self.steps.append(name)
+        return self
+
+
+class _FakeSpark:
+    def __init__(self, stream):
+        self.readStream = self
+        self._stream = stream
+
+    def format(self, _fmt):
+        return self
+
+    def option(self, _key, _value):
+        return self
+
+    def load(self, _path):
+        return self._stream
+
+
+def test_bronze_takes_file_landed_timestamp_straight_from_the_source(monkeypatch):
+    monkeypatch.setattr(main_json_utils, "resolve_autoloader_recovery", lambda _spark: False)
+    monkeypatch.setattr(main_json_utils, "col", lambda value: value)
+    stream = _FakeStream()
+
+    main_json_utils._build_autoloader_reader(
+        spark=_FakeSpark(stream),
+        source_path="s3://bucket/raw/exposure_state",
+        file_format="json",
+        schema_hints=None,
+        use_managed_file_events=True,
+        include_source_file=True,
+    )
+
+    assert stream.steps == [add_file_landed_timestamp, "source_file"]

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.2}/tests/test_transformation_module.py

@@ -6,7 +6,7 @@ from pyspark.sql import SparkSession, Row
 from pyspark.sql.functions import expr
 from pyspark.sql.types import TimestampType, StringType
 from datetime import datetime
-from dlt_utils.dlt_transformations import rename_columns, apply_partitions, update_cdc_timestamp, add_default_value_for_removed_col, rename_columns
+from dlt_utils.dlt_transformations import rename_columns, apply_partitions, update_cdc_timestamp, add_default_value_for_removed_col, rename_columns, add_file_landed_timestamp
 # Mock databricks.sdk.runtime if not in a Databricks environment.
 #
 # Catching Exception and not ImportError: importing this module builds a
@@ -79,6 +79,29 @@ def test_update_cdc_timestamp(spark):
     assert result[1]['cdc_timestamp'] == datetime(2024, 1, 1)
 
 
+def test_update_cdc_timestamp_ignores_file_landed_timestamp(spark):
+    data = [
+        Row(id=1, updated_at=datetime(2024, 1, 10), file_landed_timestamp=datetime(2024, 3, 1), cdc_timestamp=None),
+        Row(id=2, updated_at=datetime(2024, 1, 10), file_landed_timestamp=datetime(2024, 3, 1),
+            cdc_timestamp=datetime(2024, 1, 9)),
+    ]
+
+    result = update_cdc_timestamp(spark.createDataFrame(data), time_diff_threshold=5).collect()
+
+    assert result[0]['cdc_timestamp'] == datetime(2024, 1, 10)
+
+
+def test_add_file_landed_timestamp_is_the_file_modification_time(spark, tmp_path):
+    path = str(tmp_path / "landed")
+    spark.createDataFrame([Row(id=1)]).coalesce(1).write.parquet(path)
+    part = next(name for name in os.listdir(path) if name.endswith(".parquet"))
+    modified = datetime.fromtimestamp(int(os.path.getmtime(os.path.join(path, part))))
+
+    result = add_file_landed_timestamp(spark.read.parquet(path)).collect()[0]
+
+    assert result['file_landed_timestamp'].replace(microsecond=0) == modified
+
+
 
 
 def test_add_default_value_for_removed_col(spark):