dlt-utils 0.1.2__tar.gz → 0.3.0__tar.gz

{dlt_utils-0.1.2 → dlt_utils-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dlt_utils
-Version: 0.1.2
+Version: 0.3.0
 Summary: Shared utilities for dlt data pipelines with multi-company support
 Author: Jay van den Bos
 License: MIT
@@ -18,6 +18,8 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 Requires-Dist: dlt>=1.19.1
+Requires-Dist: azure-identity>=1.15.0
+Requires-Dist: azure-keyvault-secrets>=4.8.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"

{dlt_utils-0.1.2 → dlt_utils-0.3.0}/dlt_utils/__init__.py

@@ -27,8 +27,10 @@ from .args import (
     create_base_parser,
     CommonArgs,
 )
+from .azure_keyvault import AzureKeyVault
 from .logging import configure_logging
 from .pipeline import (
+    cleanup_tombstones,
     filter_resources,
     get_resource_names,
     mask_sensitive_value,
@@ -37,7 +39,7 @@ from .pipeline import (
 )
 
 
-__version__ = "0.1.2"
+__version__ = "0.3.0"
 
 __all__ = [
     # Incremental
@@ -53,9 +55,12 @@ __all__ = [
     "add_common_args",
     "CommonArgs",
     "create_base_parser",
+    # azure_keyvault
+    "AzureKeyVault",
     # logging
     "configure_logging",
     # pipeline
+    "cleanup_tombstones",
     "filter_resources",
     "get_resource_names",
     "mask_sensitive_value",

dlt_utils-0.3.0/dlt_utils/azure_keyvault.py

@@ -0,0 +1,93 @@
+"""
+Azure Key Vault utilities voor secret management.
+
+Dit module biedt een AzureKeyVault class voor het ophalen, opslaan en beheren van
+secrets in Azure Key Vault. Authenticatie gebeurt via Service Principal credentials
+(client_id, client_secret, tenant_id).
+
+Gebruik:
+    - Secrets ophalen voor database credentials, API keys, etc.
+    - Secrets opslaan/updaten vanuit pipelines
+    - Tags updaten voor secrets
+
+Vereiste environment variables:
+    - KEYVAULT_BASE_URL: URL van de Key Vault (bijv. https://myvault.vault.azure.net/)
+    - KEYVAULT_SECRET_NAME: Naam van de secret
+    - AZURE_TENANT_ID: Azure AD tenant ID
+    - AZURE_CLIENT_ID: Service Principal client ID
+    - AZURE_CLIENT_SECRET: Service Principal client secret
+"""
+from azure.identity import ClientSecretCredential
+from azure.keyvault.secrets import SecretClient
+import json
+import os
+
+
+class AzureKeyVault:
+    def __init__(self):
+        """
+        Base class for Azure
+        """
+        super(AzureKeyVault, self).__init__()
+
+        # Azure Key Vault configuration
+        self.KEYVAULT_BASE_URL = os.getenv('KEYVAULT_BASE_URL')
+        self.KEYVAULT_SECRET_NAME = os.getenv('KEYVAULT_SECRET_NAME')
+
+        # Initialize Azure credential and Key Vault client
+        self.credential = ClientSecretCredential(
+            tenant_id=os.getenv('AZURE_TENANT_ID'),
+            client_id=os.getenv('AZURE_CLIENT_ID'),
+            client_secret=os.getenv('AZURE_CLIENT_SECRET')
+        )
+        self.secret_client = SecretClient(
+            vault_url=self.KEYVAULT_BASE_URL,
+            credential=self.credential
+        )
+
+    def get_secret(self) -> object:
+        """
+        Get secret from Azure Key Vault.
+
+        Returns
+        -------
+        object
+            Object with value, properties, etc. of the secret
+        """
+        secret = self.secret_client.get_secret(self.KEYVAULT_SECRET_NAME)
+
+        return secret
+
+    def update_secret_tags(self, tags: dict):
+        """
+        Update tags on the Key Vault secret.
+
+        Parameters
+        ----------
+        tags : dict
+            Dictionary of tags to set on the secret
+        """
+        self.secret_client.update_secret_properties(
+            self.KEYVAULT_SECRET_NAME,
+            tags=tags
+        )
+        print("Updated Key Vault tags successfully")
+
+    def set_secret(self, secret_value: dict):
+        """
+        Set secret to Azure Key Vault.
+
+        Parameters
+        ----------
+        secret_value : dict
+            Secret value to set to Azure Key Vault
+        """
+
+        self.secret_client.set_secret(
+            self.KEYVAULT_SECRET_NAME,
+            json.dumps(secret_value),
+            tags={}
+        )
+        print("Secret set to Key Vault successfully")
+
+

{dlt_utils-0.1.2 → dlt_utils-0.3.0}/dlt_utils/pipeline.py

@@ -124,3 +124,48 @@ def mask_sensitive_value(value: str | None, show_chars: int = 2) -> str:
     
     mid = len(value) // 2
     return f"{value[:show_chars]}..{value[mid:mid+show_chars]}..***"
+
+
+def cleanup_tombstones(
+    pipeline: Any,
+    resources: list[str],
+    column_name: str = "_tt_is_tombstone",
+) -> None:
+    """
+    Clean up tombstone records from tables after a successful pipeline run.
+    
+    Tombstones are placeholder records used to ensure delete-insert semantics
+    work correctly for partitions that have no real data. After the pipeline
+    completes, these tombstone records should be removed.
+    
+    Args:
+        pipeline: A dlt pipeline object with sql_client() method.
+        resources: List of resource/table names to clean tombstones from.
+        column_name: The boolean column name that marks tombstone records.
+                     Defaults to "_tt_is_tombstone".
+    
+    Example:
+        # Clean up tombstones from all resources except 'customer'
+        tombstone_resources = [r for r in sync_resources if r != "customer"]
+        cleanup_tombstones(pipeline, tombstone_resources)
+        
+        # Clean up with a custom column name
+        cleanup_tombstones(pipeline, ["my_table"], column_name="is_tombstone")
+    """
+    if not resources:
+        logger.debug("No resources provided for tombstone cleanup")
+        return
+    
+    with pipeline.sql_client() as sql_client:
+        for resource_name in resources:
+            table_name = sql_client.make_qualified_table_name(resource_name)
+            delete_query = f"DELETE FROM {table_name} WHERE {column_name} = TRUE"
+            
+            try:
+                sql_client.execute_sql(delete_query)
+                logger.info(f"Cleaned up tombstone records from {table_name}")
+            except Exception as e:
+                # Table might not exist on first run or if resource wasn't selected
+                logger.warning(
+                    f"Could not clean tombstones from {table_name}: {e}"
+                )

{dlt_utils-0.1.2 → dlt_utils-0.3.0}/dlt_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dlt_utils
-Version: 0.1.2
+Version: 0.3.0
 Summary: Shared utilities for dlt data pipelines with multi-company support
 Author: Jay van den Bos
 License: MIT
@@ -18,6 +18,8 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 Requires-Dist: dlt>=1.19.1
+Requires-Dist: azure-identity>=1.15.0
+Requires-Dist: azure-keyvault-secrets>=4.8.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"

{dlt_utils-0.1.2 → dlt_utils-0.3.0}/dlt_utils.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ README.md
 pyproject.toml
 dlt_utils/__init__.py
 dlt_utils/args.py
+dlt_utils/azure_keyvault.py
 dlt_utils/dates.py
 dlt_utils/incremental.py
 dlt_utils/logging.py
@@ -12,5 +13,6 @@ dlt_utils.egg-info/SOURCES.txt
 dlt_utils.egg-info/dependency_links.txt
 dlt_utils.egg-info/requires.txt
 dlt_utils.egg-info/top_level.txt
+tests/test_azure_keyvault.py
 tests/test_dates.py
 tests/test_incremental.py

dlt_utils-0.3.0/dlt_utils.egg-info/requires.txt

@@ -0,0 +1,8 @@
+dlt>=1.19.1
+azure-identity>=1.15.0
+azure-keyvault-secrets>=4.8.0
+
+[dev]
+pytest>=7.0
+pytest-cov
+ruff

{dlt_utils-0.1.2 → dlt_utils-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dlt_utils"
-version = "0.1.2"
+version = "0.3.0"
 description = "Shared utilities for dlt data pipelines with multi-company support"
 readme = "README.md"
 license = { text = "MIT" }
@@ -25,6 +25,8 @@ classifiers = [
 ]
 dependencies = [
     "dlt>=1.19.1",
+    "azure-identity>=1.15.0",
+    "azure-keyvault-secrets>=4.8.0",
 ]
 
 [project.optional-dependencies]

dlt_utils-0.3.0/tests/test_azure_keyvault.py

@@ -0,0 +1,126 @@
+"""Tests for Azure Key Vault utilities."""
+import json
+import os
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+class TestAzureKeyVault:
+    """Test cases for AzureKeyVault class."""
+
+    @patch.dict(os.environ, {
+        'KEYVAULT_BASE_URL': 'https://test-vault.vault.azure.net/',
+        'KEYVAULT_SECRET_NAME': 'test-secret',
+        'AZURE_TENANT_ID': 'test-tenant-id',
+        'AZURE_CLIENT_ID': 'test-client-id',
+        'AZURE_CLIENT_SECRET': 'test-client-secret'
+    })
+    @patch('dlt_utils.azure_keyvault.SecretClient')
+    @patch('dlt_utils.azure_keyvault.ClientSecretCredential')
+    def test_init(self, mock_credential, mock_secret_client):
+        """Test AzureKeyVault initialization."""
+        from dlt_utils.azure_keyvault import AzureKeyVault
+        
+        kv = AzureKeyVault()
+        
+        # Verify credential was created with correct parameters
+        mock_credential.assert_called_once_with(
+            tenant_id='test-tenant-id',
+            client_id='test-client-id',
+            client_secret='test-client-secret'
+        )
+        
+        # Verify SecretClient was created
+        mock_secret_client.assert_called_once()
+        assert kv.KEYVAULT_BASE_URL == 'https://test-vault.vault.azure.net/'
+        assert kv.KEYVAULT_SECRET_NAME == 'test-secret'
+
+    @patch.dict(os.environ, {
+        'KEYVAULT_BASE_URL': 'https://test-vault.vault.azure.net/',
+        'KEYVAULT_SECRET_NAME': 'test-secret',
+        'AZURE_TENANT_ID': 'test-tenant-id',
+        'AZURE_CLIENT_ID': 'test-client-id',
+        'AZURE_CLIENT_SECRET': 'test-client-secret'
+    })
+    @patch('dlt_utils.azure_keyvault.SecretClient')
+    @patch('dlt_utils.azure_keyvault.ClientSecretCredential')
+    def test_get_secret(self, mock_credential, mock_secret_client):
+        """Test getting a secret from Key Vault."""
+        from dlt_utils.azure_keyvault import AzureKeyVault
+        
+        # Setup mock
+        mock_secret = MagicMock()
+        mock_secret.value = '{"key": "value"}'
+        mock_secret_client.return_value.get_secret.return_value = mock_secret
+        
+        kv = AzureKeyVault()
+        result = kv.get_secret()
+        
+        # Verify get_secret was called with correct secret name
+        kv.secret_client.get_secret.assert_called_once_with('test-secret')
+        assert result == mock_secret
+
+    @patch.dict(os.environ, {
+        'KEYVAULT_BASE_URL': 'https://test-vault.vault.azure.net/',
+        'KEYVAULT_SECRET_NAME': 'test-secret',
+        'AZURE_TENANT_ID': 'test-tenant-id',
+        'AZURE_CLIENT_ID': 'test-client-id',
+        'AZURE_CLIENT_SECRET': 'test-client-secret'
+    })
+    @patch('dlt_utils.azure_keyvault.SecretClient')
+    @patch('dlt_utils.azure_keyvault.ClientSecretCredential')
+    def test_update_secret_tags(self, mock_credential, mock_secret_client):
+        """Test updating secret tags."""
+        from dlt_utils.azure_keyvault import AzureKeyVault
+        
+        kv = AzureKeyVault()
+        tags = {'environment': 'test', 'version': '1.0'}
+        kv.update_secret_tags(tags)
+        
+        # Verify update_secret_properties was called
+        kv.secret_client.update_secret_properties.assert_called_once_with(
+            'test-secret',
+            tags=tags
+        )
+
+    @patch.dict(os.environ, {
+        'KEYVAULT_BASE_URL': 'https://test-vault.vault.azure.net/',
+        'KEYVAULT_SECRET_NAME': 'test-secret',
+        'AZURE_TENANT_ID': 'test-tenant-id',
+        'AZURE_CLIENT_ID': 'test-client-id',
+        'AZURE_CLIENT_SECRET': 'test-client-secret'
+    })
+    @patch('dlt_utils.azure_keyvault.SecretClient')
+    @patch('dlt_utils.azure_keyvault.ClientSecretCredential')
+    def test_set_secret(self, mock_credential, mock_secret_client):
+        """Test setting a secret in Key Vault."""
+        from dlt_utils.azure_keyvault import AzureKeyVault
+        
+        kv = AzureKeyVault()
+        secret_value = {'database': 'prod', 'password': 'secret123'}
+        kv.set_secret(secret_value)
+        
+        # Verify set_secret was called with correct parameters
+        kv.secret_client.set_secret.assert_called_once_with(
+            'test-secret',
+            json.dumps(secret_value),
+            tags={}
+        )
+
+
+class TestAzureKeyVaultMissingEnvVars:
+    """Test AzureKeyVault behavior with missing environment variables."""
+
+    @patch.dict(os.environ, {}, clear=True)
+    @patch('dlt_utils.azure_keyvault.SecretClient')
+    @patch('dlt_utils.azure_keyvault.ClientSecretCredential')
+    def test_init_missing_env_vars(self, mock_credential, mock_secret_client):
+        """Test initialization with missing environment variables."""
+        from dlt_utils.azure_keyvault import AzureKeyVault
+        
+        kv = AzureKeyVault()
+        
+        # Should initialize but with None values
+        assert kv.KEYVAULT_BASE_URL is None
+        assert kv.KEYVAULT_SECRET_NAME is None

dlt_utils-0.1.2/dlt_utils.egg-info/requires.txt

@@ -1,6 +0,0 @@
-dlt>=1.19.1
-
-[dev]
-pytest>=7.0
-pytest-cov
-ruff