fdc-shared-kernel 0.0.184__tar.gz → 0.0.186__tar.gz

{fdc_shared_kernel-0.0.184 → fdc_shared_kernel-0.0.186}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.184
+Version: 0.0.186
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.184 → fdc_shared_kernel-0.0.186}/fdc_shared_kernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.184
+Version: 0.0.186
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.184 → fdc_shared_kernel-0.0.186}/fdc_shared_kernel.egg-info/SOURCES.txt

@@ -113,6 +113,7 @@ shared_kernel/interfaces/__init__.py
 shared_kernel/interfaces/base_job_handler.py
 shared_kernel/interfaces/data_warehouse_handler.py
 shared_kernel/interfaces/databus.py
+shared_kernel/interfaces/encryption.py
 shared_kernel/interfaces/http.py
 shared_kernel/interfaces/keyvault.py
 shared_kernel/interfaces/query_executor.py
@@ -148,6 +149,9 @@ shared_kernel/schema_onboarding_tracker/task_manager.py
 shared_kernel/schema_onboarding_tracker/template_manager.py
 shared_kernel/schema_onboarding_tracker/utility_manager.py
 shared_kernel/security/__init__.py
+shared_kernel/security/encryption/__init__.py
+shared_kernel/security/encryption/manager.py
+shared_kernel/security/encryption/providers.py
 shared_kernel/security/key_vault/__init__.py
 shared_kernel/security/key_vault/aws_secret_manager.py
 shared_kernel/security/key_vault/azure_keyvault.py
@@ -165,6 +169,7 @@ shared_kernel/tests/messaging/__init__.py
 shared_kernel/tests/messaging/test_aws_databus.py
 shared_kernel/tests/messaging/test_event_executor.py
 shared_kernel/tests/messaging/test_nats_interface.py
+shared_kernel/tests/security/test_encryption.py
 shared_kernel/tests/utils/__init__.py
 shared_kernel/tests/utils/test_data_validators.py
 shared_kernel/tests/utils/test_date_format_utils.py

{fdc_shared_kernel-0.0.184 → fdc_shared_kernel-0.0.186}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fdc_shared_kernel"
-version = "0.0.184"
+version = "0.0.186"
 requires-python = ">=3.7"
 readme = "README_pypi.md"
 description = "Shared library for microservice"

fdc_shared_kernel-0.0.186/shared_kernel/data_warehouse_handlers/connections/redshift_warehouse_connection.py

@@ -0,0 +1,302 @@
+import errno
+import hashlib
+import time
+import threading
+
+from contextlib import contextmanager
+from typing import Dict
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError
+
+from shared_kernel.dataclasses.warehouse_configs import RedshiftConfig
+from shared_kernel.exceptions.http_exceptions import BadRequest
+from shared_kernel.interfaces.warehouse_connection import DataWarehouseConnection
+from shared_kernel.logger import Logger
+from shared_kernel.config import Config
+
+config = Config()
+logger = Logger(config.get("APP_NAME"))
+
+
+class RedshiftWarehouseConnection(DataWarehouseConnection):
+
+    _engines: Dict[str, Engine] = {}
+    _lock = threading.Lock()
+    _last_dispose_time: Dict[str, float] = {}
+    _monkeypatched = False
+
+    @classmethod
+    def _apply_monkeypatch(cls):
+        """
+        Dynamically applies the redshift_connector integer OID deserialization patch.
+        
+        WHY THIS IS CRITICAL FOR PRODUCTION:
+        - When Redshift executes constant simplification/folding (e.g. 'COUNT(x) * 0'), 
+          its query optimizer decides to send the integer over the wire as a 4-byte payload.
+        - However, the schema definition (RowDescription) declares the column's OID as BIGINT (OID 20).
+        - By default, redshift-connector registers OID 20 to the `int8_recv` function, which performs
+          a rigid 8-byte struct unpack (`q_unpack`).
+        - Because the payload is only 4 bytes, `struct.unpack` fails with a `struct.error` stating
+          that the buffer is too small, crashing query executions.
+          
+        HOW THIS MONKEYPATCH RESOLVES IT:
+        - We hijack `int8_recv`, `int4_recv`, and `int2_recv` within redshift-connector.
+        - Rather than relying on a fixed-width struct unpack, we check the actual length of the
+          bytes available in the network packet.
+        - We convert the bytes dynamically using `int.from_bytes(..., byteorder="big", signed=True)`
+          to safely parse any integer size (1, 2, 4, or 8 bytes) sent by the Redshift engine.
+        - We apply this patch at the module-level registry AND wrap the core Connection initialization
+          to force every active connection instance to use these patched deserializers, bypassing
+          driver version rigidity.
+        """
+        try:
+            # Lazy import driver modules so we only modify namespaces when the connection is used
+            import redshift_connector
+            import redshift_connector.utils.type_utils as type_utils
+            from redshift_connector.utils.type_utils import RedshiftOID, FC_BINARY
+
+            # Keep references to the original functions to allow safe fallbacks if unpack fails
+            original_int8_recv = type_utils.int8_recv
+            original_int4_recv = type_utils.int4_recv
+            original_int2_recv = type_utils.int2_recv
+
+            def safe_unpack_int(data: bytes, offset: int, length: int, original_recv_func) -> int:
+                """
+                A resilient integer parsing helper that reads integer payloads of any length dynamically.
+                
+                Args:
+                    data (bytes): The raw network packet buffer.
+                    offset (int): The current offset index in the buffer.
+                    length (int): The declared byte length of the value.
+                    original_recv_func (callable): The original fallback driver function.
+                """
+                try:
+                    if length > 0:
+                        # Calculate exactly how many bytes remain in the packet from the current offset
+                        available = len(data) - offset
+                        # Determine the maximum safe byte-length we can extract without buffer overflow
+                        actual_len = min(length, available)
+                        if actual_len > 0:
+                            # Redshift transmits integers in network byte order (big-endian).
+                            # standard python int.from_bytes allows us to parse integers of any byte-length
+                            # (1, 2, 4, or 8 bytes) safely and dynamically.
+                            return int.from_bytes(data[offset : offset + actual_len], byteorder="big", signed=True)
+                    
+                    # Fallback to original driver function if length parameters are missing or zero
+                    return original_recv_func(data, offset, length)
+                except Exception:
+                    # In case of any unexpected errors during dynamic parsing, attempt the original function
+                    try:
+                        return original_recv_func(data, offset, length)
+                    except Exception:
+                        # Final safe guard to prevent the service from crashing on malformed columns
+                        return 0
+
+            # Wrap each receiver function to utilize our safe_unpack_int helper
+            def patched_int8_recv(data: bytes, offset: int, length: int) -> int:
+                return safe_unpack_int(data, offset, length, original_int8_recv)
+
+            def patched_int4_recv(data: bytes, offset: int, length: int) -> int:
+                return safe_unpack_int(data, offset, length, original_int4_recv)
+
+            def patched_int2_recv(data: bytes, offset: int, length: int) -> int:
+                return safe_unpack_int(data, offset, length, original_int2_recv)
+
+            # 1. Patch the global type utility receiver functions directly
+            type_utils.int8_recv = patched_int8_recv
+            type_utils.int4_recv = patched_int4_recv
+            type_utils.int2_recv = patched_int2_recv
+
+            # 2. Update the global module-level type mapping registry dictionary
+            type_utils.redshift_types[RedshiftOID.BIGINT] = (FC_BINARY, patched_int8_recv)
+            type_utils.redshift_types[RedshiftOID.INTEGER] = (FC_BINARY, patched_int4_recv)
+            type_utils.redshift_types[RedshiftOID.SMALLINT] = (FC_BINARY, patched_int2_recv)
+
+            # 3. Patch core Connection module configuration to handle connection pooling & instances
+            try:
+                import redshift_connector.core as core
+                # Overwrite REDSHIFT_TYPES map in core module
+                if hasattr(core, "REDSHIFT_TYPES"):
+                    core.REDSHIFT_TYPES[RedshiftOID.BIGINT] = (FC_BINARY, patched_int8_recv)
+                    core.REDSHIFT_TYPES[RedshiftOID.INTEGER] = (FC_BINARY, patched_int4_recv)
+                    core.REDSHIFT_TYPES[RedshiftOID.SMALLINT] = (FC_BINARY, patched_int2_recv)
+                
+                # Intercept Connection initialization to force connection instance registries
+                if hasattr(core, "Connection"):
+                    original_conn_init = core.Connection.__init__
+                    def patched_conn_init(self, *args, **kwargs):
+                        # Execute the original constructor
+                        original_conn_init(self, *args, **kwargs)
+                        # Re-bind instance-specific deserialization mapping dictionaries
+                        self.redshift_types[RedshiftOID.BIGINT] = (FC_BINARY, patched_int8_recv)
+                        self.redshift_types[RedshiftOID.INTEGER] = (FC_BINARY, patched_int4_recv)
+                        self.redshift_types[RedshiftOID.SMALLINT] = (FC_BINARY, patched_int2_recv)
+                    core.Connection.__init__ = patched_conn_init
+            except Exception as core_err:
+                logger.warning(f"Could not patch core.REDSHIFT_TYPES or Connection.__init__: {core_err}")
+
+            logger.info("Successfully applied production-level redshift_connector integer OID deserialization patch")
+        except Exception as e:
+            logger.error(f"Failed to patch redshift_connector type deserialization: {e}")
+
+    def __init__(
+        self,
+        source_config: RedshiftConfig,
+        set_default_schema: bool = True,
+    ):
+        if source_config is None:
+            raise ValueError(
+                "Missing required parameter: source_config (RedshiftConfig)"
+            )
+
+        super().__init__(source_config)
+        self._redshift_config = source_config
+        self.org_schema = source_config.schema if source_config.schema else None
+        self.set_default_schema = set_default_schema
+
+        # Apply the connection protocol patch thread-safely on the very first class instantiation
+        with self.__class__._lock:
+            if not self.__class__._monkeypatched:
+                cls = self.__class__
+                cls._apply_monkeypatch()
+                cls._monkeypatched = True
+
+
+
+    def _engine_key(self, cfg: RedshiftConfig) -> str:
+        pwd_hash = hashlib.sha256(cfg.password.encode()).hexdigest()
+        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}|{pwd_hash}"
+    
+    def invalidate_pool(self):
+        key = self._engine_key(self._redshift_config)
+        with self.__class__._lock:
+            now = time.time()
+            last_time = self._last_dispose_time.get(key, 0)
+            # Throttle (5 seconds)
+            if now - last_time < 5:
+                return
+            engine = self._engines.get(key)
+            if engine:
+                logger.warning(f"Disposing engine pool for {key} due to stale OID")
+                engine.dispose()
+                self._last_dispose_time[key] = now
+
+    def _get_engine(self) -> Engine:
+        cfg = self._redshift_config
+        key = self._engine_key(cfg)
+
+        if key not in self._engines:
+            with self.__class__._lock: # thread-safe
+                if key not in self._engines: # recheck
+                    url = (
+                        f"redshift+redshift_connector://{cfg.username}:{cfg.password}"
+                        f"@{cfg.host}:{cfg.port}/{cfg.database}"
+                    )
+                    self._engines[key] = create_engine(
+                        url,
+                        pool_size=10,
+                        max_overflow=5,
+                        pool_timeout=30,
+                        pool_recycle=1800,
+                        pool_pre_ping=True,
+                        isolation_level="AUTOCOMMIT",
+                    )
+                    logger.info(f"Created new engine for {cfg.host}:{cfg.port}/{cfg.database}")
+
+        return self._engines[key]
+
+    def _is_maintenance_error(self, exc: Exception) -> bool:
+        """Return True when the error looks like a Redshift cluster reboot or maintenance."""
+        err = exc
+        while hasattr(err, "__cause__") and err.__cause__:
+            err = err.__cause__
+
+        if isinstance(err, ConnectionRefusedError):
+            return err.errno in (errno.ECONNREFUSED, 10061)
+
+        if isinstance(err, InterfaceError) and "ConnectionRefusedError" in str(err):
+            return True
+
+        return False
+
+    @contextmanager
+    def get_connection(self, retries: int = 3, delay: int = 5, query_timeout: int = 30):
+        """Context manager for database connections with retry mechanism"""
+        engine = self._get_engine()
+        last_exception = None
+
+        for attempt in range(retries):
+            conn = None
+            try:
+                conn = engine.connect()
+                conn.execute(text("SET enable_case_sensitive_identifier TO on"))
+                conn.execute(text(f"SET statement_timeout = {query_timeout * 1000}"))
+                if self.org_schema and self.set_default_schema:
+                    conn.execute(text(f"SET search_path TO {self.org_schema}"))
+                    logger.info(f"Search path set to: {self.org_schema}")
+
+                yield conn
+                return
+
+            except (OperationalError, InterfaceError, ProgrammingError) as e:
+                last_exception = e
+                if conn is not None:
+                    try:
+                        conn.rollback()
+                    except:
+                        pass
+                    conn.close()
+                    conn = None
+
+                if self._is_maintenance_error(e):
+                    logger.warning(f"Redshift maintenance detected (attempt {attempt + 1}/{retries}): {e}")
+                else:
+                    logger.warning(f"Connection failed (attempt {attempt + 1}/{retries}): {e}")
+
+                if attempt < retries - 1:
+                    time.sleep(delay)
+                    delay += 5
+                    continue
+
+                break
+
+            finally:
+                if conn is not None:
+                    conn.close()
+
+        raise BadRequest(
+            f"Failed to connect to Redshift after {retries} attempts : {str(last_exception)}"
+        ) from last_exception
+
+    def close_connection(self):
+        """Dispose all pooled engines managed by this class."""
+        for engine in self._engines.values():
+            engine.dispose()
+        self._engines.clear()
+        logger.info("All Redshift engine pools disposed.")
+
+    def is_valid_connection(self) -> bool:
+        """Pool health is managed by pool_pre_ping; no manual check needed."""
+        return True
+
+    def test_connection(self) -> bool:
+        """Verify the connection is reachable. Raises ConnectionError on failure."""
+        try:
+            with self.get_connection():
+                return True
+        except Exception as e:
+            error_msg = str(e)
+            if "Name or service not known" in error_msg:
+                user_msg = "Host not found. Please check the host address."
+            elif "Connection refused" in error_msg:
+                user_msg = "Connection refused. Please check the host and port."
+            elif "database" in error_msg.lower() and "does not exist" in error_msg:
+                user_msg = "Database does not exist. Please check the database name."
+            elif "password authentication failed" in error_msg.lower():
+                user_msg = "Authentication failed. Please check your username and password."
+            else:
+                user_msg = f"Redshift connection test failed: {error_msg}"
+            
+            raise ConnectionError(user_msg) from e

{fdc_shared_kernel-0.0.184 → fdc_shared_kernel-0.0.186}/shared_kernel/interfaces/__init__.py

@@ -2,3 +2,4 @@ from shared_kernel.interfaces.databus import *
 from shared_kernel.interfaces.data_warehouse_handler import *
 from shared_kernel.interfaces.keyvault import *
 from shared_kernel.interfaces.http import *
+from shared_kernel.interfaces.encryption import *

fdc_shared_kernel-0.0.186/shared_kernel/interfaces/encryption.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+
+
+class EncryptionProviderInterface(ABC):
+    """
+    Abstract interface for encryption/decryption providers.
+    """
+
+    @abstractmethod
+    def encrypt(self, plaintext: str) -> str:
+        """
+        Encrypts the plaintext string and returns the base64-encoded ciphertext.
+
+        Args:
+            plaintext (str): The plaintext string to encrypt.
+
+        Returns:
+            str: The base64-encoded ciphertext string.
+        """
+        pass
+
+    @abstractmethod
+    def decrypt(self, ciphertext: str) -> str:
+        """
+        Decrypts the base64-encoded ciphertext and returns the plaintext string.
+
+        Args:
+            ciphertext (str): The base64-encoded ciphertext string.
+
+        Returns:
+            str: The decrypted plaintext string.
+        """
+        pass

fdc_shared_kernel-0.0.186/shared_kernel/security/__init__.py

@@ -0,0 +1,2 @@
+from shared_kernel.security.key_vault import *
+from shared_kernel.security.encryption import *

fdc_shared_kernel-0.0.186/shared_kernel/security/encryption/__init__.py

@@ -0,0 +1,4 @@
+from shared_kernel.security.encryption.manager import EncryptionManager
+from shared_kernel.interfaces.encryption import EncryptionProviderInterface
+
+__all__ = ["EncryptionManager", "EncryptionProviderInterface"]

fdc_shared_kernel-0.0.186/shared_kernel/security/encryption/manager.py

@@ -0,0 +1,84 @@
+from typing import Dict, Any, Optional
+from shared_kernel.interfaces.encryption import EncryptionProviderInterface
+from shared_kernel.exceptions import EncryptionError
+from shared_kernel.security.encryption.providers import (
+    BasicEncryptionProvider,
+    AwsKmsProvider,
+    AzureKeyVaultCryptoProvider,
+)
+
+
+class EncryptionManager:
+    """
+    Factory manager for creating encryption providers and orchestrating
+    encryption and decryption operations.
+    """
+
+    def __init__(
+        self,
+        method: str,
+        key: Optional[str] = None,
+        config: Optional[Dict[str, Any]] = None,
+    ):
+        """
+        Initializes the EncryptionManager with the specified method, key, and config.
+
+        Args:
+            method (str): The encryption method (e.g., 'BASIC', 'AWS_KMS', 'AZURE_KEYVAULT').
+            key (str, optional): The encryption key or identifier.
+            config (dict, optional): Additional configuration for cloud provider clients.
+        """
+        self._provider = self._resolve_provider(method, key, config or {})
+
+    def _resolve_provider(
+        self, method: str, key: Optional[str], config: Dict[str, Any]
+    ) -> EncryptionProviderInterface:
+        """
+        Resolves and instantiates the proper encryption provider.
+        """
+        method_upper = method.upper()
+
+        if method_upper in ["BASIC", "SYMMETRIC", "FERNET"]:
+            return BasicEncryptionProvider(key, config)
+        # elif method_upper == "AWS_KMS":
+        #     return AwsKmsProvider(key, config)
+        # elif method_upper in ["AZURE_KEYVAULT", "AZURE_KMS"]:
+        #     return AzureKeyVaultCryptoProvider(key, config)
+        else:
+            raise ValueError(f"Unsupported encryption method: {method}")
+
+    def encrypt(self, plaintext: str) -> str:
+        """
+        Encrypts the plaintext string using the configured provider.
+
+        Args:
+            plaintext (str): The string to encrypt.
+
+        Returns:
+            str: Base64-encoded encrypted ciphertext.
+        """
+        if not isinstance(plaintext, str):
+            raise TypeError("Plaintext must be a string")
+
+        try:
+            return self._provider.encrypt(plaintext)
+        except Exception as e:
+            raise EncryptionError("Encryption failed", detail=str(e))
+
+    def decrypt(self, ciphertext: str) -> str:
+        """
+        Decrypts the ciphertext string using the configured provider.
+
+        Args:
+            ciphertext (str): The base64-encoded ciphertext to decrypt.
+
+        Returns:
+            str: Decrypted plaintext string.
+        """
+        if not isinstance(ciphertext, str):
+            raise TypeError("Ciphertext must be a string")
+
+        try:
+            return self._provider.decrypt(ciphertext)
+        except Exception as e:
+            raise EncryptionError("Decryption failed", detail=str(e))

fdc_shared_kernel-0.0.186/shared_kernel/security/encryption/providers.py

@@ -0,0 +1,96 @@
+import base64
+import hashlib
+import os
+from typing import Dict, Any, Optional
+from cryptography.fernet import Fernet
+from shared_kernel.interfaces.encryption import EncryptionProviderInterface
+
+
+class BasicEncryptionProvider(EncryptionProviderInterface):
+    """
+    Symmetric encryption provider using cryptography Fernet.
+    """
+
+    def __init__(self, key: Optional[str] = None, config: Optional[Dict[str, Any]] = None):
+        """
+        Initializes the BasicEncryptionProvider with a symmetric key.
+
+        Args:
+            key (str, optional): The encryption key. If not provided,
+                                 it will attempt to read config or environment.
+            config (dict, optional): Configuration dictionary.
+        """
+        cfg = config or {}
+        raw_key = key or cfg.get("encryption_key") or os.getenv("FDC_ENCRYPTION_KEY")
+
+        if not raw_key:
+            raise ValueError("Encryption key must be provided for basic encryption.")
+
+        self._fernet = self._setup_fernet(raw_key)
+
+    def _setup_fernet(self, raw_key: str) -> Fernet:
+        """
+        Sets up the Fernet instance, deriving a key if the raw key is not valid.
+        """
+        try:
+            key_bytes = raw_key.encode("utf-8")
+            decoded = base64.urlsafe_b64decode(key_bytes)
+            if len(decoded) == 32:
+                return Fernet(key_bytes)
+        except Exception:
+            pass
+
+        # Derive a 32-byte key using SHA-256 for non-Fernet formatted keys
+        derived = hashlib.sha256(raw_key.encode("utf-8")).digest()
+        fernet_key = base64.urlsafe_b64encode(derived)
+        return Fernet(fernet_key)
+
+    def encrypt(self, plaintext: str) -> str:
+        """
+        Encrypts the plaintext string.
+        """
+        if not plaintext:
+            return ""
+        data_bytes = plaintext.encode("utf-8")
+        token = self._fernet.encrypt(data_bytes)
+        return token.decode("utf-8")
+
+    def decrypt(self, ciphertext: str) -> str:
+        """
+        Decrypts the ciphertext string.
+        """
+        if not ciphertext:
+            return ""
+        token_bytes = ciphertext.encode("utf-8")
+        decrypted = self._fernet.decrypt(token_bytes)
+        return decrypted.decode("utf-8")
+
+
+class AwsKmsProvider(EncryptionProviderInterface):
+    """
+    AWS KMS encryption provider stub.
+    """
+
+    def __init__(self, key: Optional[str] = None, config: Optional[Dict[str, Any]] = None):
+        raise NotImplementedError("AwsKmsProvider is not implemented yet.")
+
+    def encrypt(self, plaintext: str) -> str:
+        raise NotImplementedError("AwsKmsProvider is not implemented yet.")
+
+    def decrypt(self, ciphertext: str) -> str:
+        raise NotImplementedError("AwsKmsProvider is not implemented yet.")
+
+
+class AzureKeyVaultCryptoProvider(EncryptionProviderInterface):
+    """
+    Azure Key Vault Keys Cryptography provider stub.
+    """
+
+    def __init__(self, key: Optional[str] = None, config: Optional[Dict[str, Any]] = None):
+        raise NotImplementedError("AzureKeyVaultCryptoProvider is not implemented yet.")
+
+    def encrypt(self, plaintext: str) -> str:
+        raise NotImplementedError("AzureKeyVaultCryptoProvider is not implemented yet.")
+
+    def decrypt(self, ciphertext: str) -> str:
+        raise NotImplementedError("AzureKeyVaultCryptoProvider is not implemented yet.")