dlt-utils 0.2.0__tar.gz → 0.3.0__tar.gz

{dlt_utils-0.2.0 → dlt_utils-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dlt_utils
-Version: 0.2.0
+Version: 0.3.0
 Summary: Shared utilities for dlt data pipelines with multi-company support
 Author: Jay van den Bos
 License: MIT

{dlt_utils-0.2.0 → dlt_utils-0.3.0}/dlt_utils/__init__.py

@@ -30,6 +30,7 @@ from .args import (
 from .azure_keyvault import AzureKeyVault
 from .logging import configure_logging
 from .pipeline import (
+    cleanup_tombstones,
     filter_resources,
     get_resource_names,
     mask_sensitive_value,
@@ -38,7 +39,7 @@ from .pipeline import (
 )
 
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 
 __all__ = [
     # Incremental
@@ -59,6 +60,7 @@ __all__ = [
     # logging
     "configure_logging",
     # pipeline
+    "cleanup_tombstones",
     "filter_resources",
     "get_resource_names",
     "mask_sensitive_value",

{dlt_utils-0.2.0 → dlt_utils-0.3.0}/dlt_utils/pipeline.py

@@ -124,3 +124,48 @@ def mask_sensitive_value(value: str | None, show_chars: int = 2) -> str:
     
     mid = len(value) // 2
     return f"{value[:show_chars]}..{value[mid:mid+show_chars]}..***"
+
+
+def cleanup_tombstones(
+    pipeline: Any,
+    resources: list[str],
+    column_name: str = "_tt_is_tombstone",
+) -> None:
+    """
+    Clean up tombstone records from tables after a successful pipeline run.
+    
+    Tombstones are placeholder records used to ensure delete-insert semantics
+    work correctly for partitions that have no real data. After the pipeline
+    completes, these tombstone records should be removed.
+    
+    Args:
+        pipeline: A dlt pipeline object with sql_client() method.
+        resources: List of resource/table names to clean tombstones from.
+        column_name: The boolean column name that marks tombstone records.
+                     Defaults to "_tt_is_tombstone".
+    
+    Example:
+        # Clean up tombstones from all resources except 'customer'
+        tombstone_resources = [r for r in sync_resources if r != "customer"]
+        cleanup_tombstones(pipeline, tombstone_resources)
+        
+        # Clean up with a custom column name
+        cleanup_tombstones(pipeline, ["my_table"], column_name="is_tombstone")
+    """
+    if not resources:
+        logger.debug("No resources provided for tombstone cleanup")
+        return
+    
+    with pipeline.sql_client() as sql_client:
+        for resource_name in resources:
+            table_name = sql_client.make_qualified_table_name(resource_name)
+            delete_query = f"DELETE FROM {table_name} WHERE {column_name} = TRUE"
+            
+            try:
+                sql_client.execute_sql(delete_query)
+                logger.info(f"Cleaned up tombstone records from {table_name}")
+            except Exception as e:
+                # Table might not exist on first run or if resource wasn't selected
+                logger.warning(
+                    f"Could not clean tombstones from {table_name}: {e}"
+                )

{dlt_utils-0.2.0 → dlt_utils-0.3.0}/dlt_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dlt_utils
-Version: 0.2.0
+Version: 0.3.0
 Summary: Shared utilities for dlt data pipelines with multi-company support
 Author: Jay van den Bos
 License: MIT

{dlt_utils-0.2.0 → dlt_utils-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dlt_utils"
-version = "0.2.0"
+version = "0.3.0"
 description = "Shared utilities for dlt data pipelines with multi-company support"
 readme = "README.md"
 license = { text = "MIT" }