dlt-utils-lib 1.3.0__tar.gz → 1.3.1__tar.gz

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt_utils_lib
-Version: 1.3.0
+Version: 1.3.1
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.1}/dlt_utils/main_cdc_utils.py

@@ -39,22 +39,29 @@ def _pick_latest_source_file(spark, files_path: str) -> Optional[str]:
 def base_cdc_replication_process(dlt,
                                  spark,
                                  cdc_tables_map: list,
-                                 bronze_directory: str,
-                                 bucket_name: str,
+                                 bronze_directory: Optional[str] = None,
+                                 bucket_name: Optional[str] = None,
                                  time_diff_history_cdc_timestamp: int = 30,
                                  bronze_schema: Optional[str] = None,
                                  silver_schema: Optional[str] = None,
-                                 use_managed_file_events: bool = False):
+                                 use_managed_file_events: bool = False,
+                                 volume_source_path: Optional[str] = None):
+    if volume_source_path is None and (not bronze_directory or not bucket_name):
+        raise ValueError("Provide volume_source_path or both bucket_name and bronze_directory")
+
     # parse list with tables for replication
     for table in cdc_tables_map:
         source_table_name = table['source_table_name']
-        source_schema = table['source_schema']
         file_format = table['format']
         keys = table['keys']
         exclude_columns = table['exclude_columns']
         enable_truncate = table['enable_truncate']
-        s3_files_path = f"{source_schema}/{source_table_name}/"
-        bronze_table_path = f"{bucket_name}/{bronze_directory}/{s3_files_path}"
+        # The volume path already includes the source schema.
+        if volume_source_path is not None:
+            bronze_table_path = f"{volume_source_path.rstrip('/')}/{source_table_name}/"
+        else:
+            source_schema = table['source_schema']
+            bronze_table_path = f"{bucket_name}/{bronze_directory}/{source_schema}/{source_table_name}/"
 
         # Optional parameters
         default_value_for_removed_col = table.get('default_value_for_removed_col', {})

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.1}/dlt_utils_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dlt-utils-lib
-Version: 1.3.0
+Version: 1.3.1
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN

{dlt_utils_lib-1.3.0 → dlt_utils_lib-1.3.1}/tests/test_main_cdc_utils.py

@@ -1,3 +1,5 @@
+import pytest
+
 import dlt_utils.main_cdc_utils as main_cdc_utils
 
 
@@ -259,3 +261,49 @@ def test_create_bronze_table_definition_falls_back_to_directory_when_listing_emp
     )
 
     assert load_paths == [directory]
+
+
+@pytest.mark.parametrize("source_options, expected", [
+    ({"bucket_name": "s3://bucket", "bronze_directory": "dms_bronze"},
+     "s3://bucket/dms_bronze/bifrost/customer/"),
+    ({"volume_source_path": "/Volumes/artemis_prod/stage/dms_raw/"},
+     "/Volumes/artemis_prod/stage/dms_raw/customer/"),
+    ({"volume_source_path": "/Volumes/artemis_prod/stage/dms_raw",
+      "bucket_name": "s3://bucket", "bronze_directory": "dms_bronze"},
+     "/Volumes/artemis_prod/stage/dms_raw/customer/"),
+])
+def test_base_cdc_source_paths(monkeypatch, source_options, expected):
+    captured = []
+    monkeypatch.setattr(main_cdc_utils, "create_bronze_table_definition", lambda **kw: captured.append(kw))
+    monkeypatch.setattr(main_cdc_utils, "silver_streaming_process", lambda **kw: None)
+    table = _build_cdc_table()
+    if "volume_source_path" in source_options:
+        del table["source_schema"]
+    main_cdc_utils.base_cdc_replication_process(
+        dlt=object(), spark=object(), cdc_tables_map=[table],
+        **source_options,
+    )
+    assert captured[0]["files_path"] == expected
+
+
+@pytest.mark.parametrize("source_options", [
+    {},
+    {"bucket_name": "s3://bucket"},
+    {"bronze_directory": "dms_bronze"},
+])
+def test_base_cdc_requires_complete_source_configuration(source_options):
+    with pytest.raises(ValueError, match="Provide volume_source_path or both bucket_name and bronze_directory"):
+        main_cdc_utils.base_cdc_replication_process(
+            dlt=object(), spark=object(), cdc_tables_map=[_build_cdc_table()],
+            **source_options,
+        )
+
+
+def test_base_cdc_s3_requires_source_schema():
+    table = _build_cdc_table()
+    del table["source_schema"]
+    with pytest.raises(KeyError, match="source_schema"):
+        main_cdc_utils.base_cdc_replication_process(
+            dlt=object(), spark=object(), cdc_tables_map=[table],
+            bucket_name="s3://bucket", bronze_directory="dms_bronze",
+        )