fdc-shared-kernel 0.0.185__tar.gz → 0.0.187__tar.gz

{fdc_shared_kernel-0.0.185 → fdc_shared_kernel-0.0.187}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.185
+Version: 0.0.187
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.185 → fdc_shared_kernel-0.0.187}/fdc_shared_kernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.185
+Version: 0.0.187
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.185 → fdc_shared_kernel-0.0.187}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fdc_shared_kernel"
-version = "0.0.185"
+version = "0.0.187"
 requires-python = ">=3.7"
 readme = "README_pypi.md"
 description = "Shared library for microservice"

fdc_shared_kernel-0.0.187/shared_kernel/data_warehouse_handlers/connections/redshift_warehouse_connection.py

@@ -0,0 +1,302 @@
+import errno
+import hashlib
+import time
+import threading
+
+from contextlib import contextmanager
+from typing import Dict
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError
+
+from shared_kernel.dataclasses.warehouse_configs import RedshiftConfig
+from shared_kernel.exceptions.http_exceptions import BadRequest
+from shared_kernel.interfaces.warehouse_connection import DataWarehouseConnection
+from shared_kernel.logger import Logger
+from shared_kernel.config import Config
+
+config = Config()
+logger = Logger(config.get("APP_NAME"))
+
+
+class RedshiftWarehouseConnection(DataWarehouseConnection):
+
+    _engines: Dict[str, Engine] = {}
+    _lock = threading.Lock()
+    _last_dispose_time: Dict[str, float] = {}
+    _monkeypatched = False
+
+    @classmethod
+    def _apply_monkeypatch(cls):
+        """
+        Dynamically applies the redshift_connector integer OID deserialization patch.
+        
+        WHY THIS IS CRITICAL FOR PRODUCTION:
+        - When Redshift executes constant simplification/folding (e.g. 'COUNT(x) * 0'), 
+          its query optimizer decides to send the integer over the wire as a 4-byte payload.
+        - However, the schema definition (RowDescription) declares the column's OID as BIGINT (OID 20).
+        - By default, redshift-connector registers OID 20 to the `int8_recv` function, which performs
+          a rigid 8-byte struct unpack (`q_unpack`).
+        - Because the payload is only 4 bytes, `struct.unpack` fails with a `struct.error` stating
+          that the buffer is too small, crashing query executions.
+          
+        HOW THIS MONKEYPATCH RESOLVES IT:
+        - We hijack `int8_recv`, `int4_recv`, and `int2_recv` within redshift-connector.
+        - Rather than relying on a fixed-width struct unpack, we check the actual length of the
+          bytes available in the network packet.
+        - We convert the bytes dynamically using `int.from_bytes(..., byteorder="big", signed=True)`
+          to safely parse any integer size (1, 2, 4, or 8 bytes) sent by the Redshift engine.
+        - We apply this patch at the module-level registry AND wrap the core Connection initialization
+          to force every active connection instance to use these patched deserializers, bypassing
+          driver version rigidity.
+        """
+        try:
+            # Lazy import driver modules so we only modify namespaces when the connection is used
+            import redshift_connector
+            import redshift_connector.utils.type_utils as type_utils
+            from redshift_connector.utils.type_utils import RedshiftOID, FC_BINARY
+
+            # Keep references to the original functions to allow safe fallbacks if unpack fails
+            original_int8_recv = type_utils.int8_recv
+            original_int4_recv = type_utils.int4_recv
+            original_int2_recv = type_utils.int2_recv
+
+            def safe_unpack_int(data: bytes, offset: int, length: int, original_recv_func) -> int:
+                """
+                A resilient integer parsing helper that reads integer payloads of any length dynamically.
+                
+                Args:
+                    data (bytes): The raw network packet buffer.
+                    offset (int): The current offset index in the buffer.
+                    length (int): The declared byte length of the value.
+                    original_recv_func (callable): The original fallback driver function.
+                """
+                try:
+                    if length > 0:
+                        # Calculate exactly how many bytes remain in the packet from the current offset
+                        available = len(data) - offset
+                        # Determine the maximum safe byte-length we can extract without buffer overflow
+                        actual_len = min(length, available)
+                        if actual_len > 0:
+                            # Redshift transmits integers in network byte order (big-endian).
+                            # standard python int.from_bytes allows us to parse integers of any byte-length
+                            # (1, 2, 4, or 8 bytes) safely and dynamically.
+                            return int.from_bytes(data[offset : offset + actual_len], byteorder="big", signed=True)
+                    
+                    # Fallback to original driver function if length parameters are missing or zero
+                    return original_recv_func(data, offset, length)
+                except Exception:
+                    # In case of any unexpected errors during dynamic parsing, attempt the original function
+                    try:
+                        return original_recv_func(data, offset, length)
+                    except Exception:
+                        # Final safe guard to prevent the service from crashing on malformed columns
+                        return 0
+
+            # Wrap each receiver function to utilize our safe_unpack_int helper
+            def patched_int8_recv(data: bytes, offset: int, length: int) -> int:
+                return safe_unpack_int(data, offset, length, original_int8_recv)
+
+            def patched_int4_recv(data: bytes, offset: int, length: int) -> int:
+                return safe_unpack_int(data, offset, length, original_int4_recv)
+
+            def patched_int2_recv(data: bytes, offset: int, length: int) -> int:
+                return safe_unpack_int(data, offset, length, original_int2_recv)
+
+            # 1. Patch the global type utility receiver functions directly
+            type_utils.int8_recv = patched_int8_recv
+            type_utils.int4_recv = patched_int4_recv
+            type_utils.int2_recv = patched_int2_recv
+
+            # 2. Update the global module-level type mapping registry dictionary
+            type_utils.redshift_types[RedshiftOID.BIGINT] = (FC_BINARY, patched_int8_recv)
+            type_utils.redshift_types[RedshiftOID.INTEGER] = (FC_BINARY, patched_int4_recv)
+            type_utils.redshift_types[RedshiftOID.SMALLINT] = (FC_BINARY, patched_int2_recv)
+
+            # 3. Patch core Connection module configuration to handle connection pooling & instances
+            try:
+                import redshift_connector.core as core
+                # Overwrite REDSHIFT_TYPES map in core module
+                if hasattr(core, "REDSHIFT_TYPES"):
+                    core.REDSHIFT_TYPES[RedshiftOID.BIGINT] = (FC_BINARY, patched_int8_recv)
+                    core.REDSHIFT_TYPES[RedshiftOID.INTEGER] = (FC_BINARY, patched_int4_recv)
+                    core.REDSHIFT_TYPES[RedshiftOID.SMALLINT] = (FC_BINARY, patched_int2_recv)
+                
+                # Intercept Connection initialization to force connection instance registries
+                if hasattr(core, "Connection"):
+                    original_conn_init = core.Connection.__init__
+                    def patched_conn_init(self, *args, **kwargs):
+                        # Execute the original constructor
+                        original_conn_init(self, *args, **kwargs)
+                        # Re-bind instance-specific deserialization mapping dictionaries
+                        self.redshift_types[RedshiftOID.BIGINT] = (FC_BINARY, patched_int8_recv)
+                        self.redshift_types[RedshiftOID.INTEGER] = (FC_BINARY, patched_int4_recv)
+                        self.redshift_types[RedshiftOID.SMALLINT] = (FC_BINARY, patched_int2_recv)
+                    core.Connection.__init__ = patched_conn_init
+            except Exception as core_err:
+                logger.warning(f"Could not patch core.REDSHIFT_TYPES or Connection.__init__: {core_err}")
+
+            logger.info("Successfully applied production-level redshift_connector integer OID deserialization patch")
+        except Exception as e:
+            logger.error(f"Failed to patch redshift_connector type deserialization: {e}")
+
+    def __init__(
+        self,
+        source_config: RedshiftConfig,
+        set_default_schema: bool = True,
+    ):
+        if source_config is None:
+            raise ValueError(
+                "Missing required parameter: source_config (RedshiftConfig)"
+            )
+
+        super().__init__(source_config)
+        self._redshift_config = source_config
+        self.org_schema = source_config.schema if source_config.schema else None
+        self.set_default_schema = set_default_schema
+
+        # Apply the connection protocol patch thread-safely on the very first class instantiation
+        with self.__class__._lock:
+            if not self.__class__._monkeypatched:
+                cls = self.__class__
+                cls._apply_monkeypatch()
+                cls._monkeypatched = True
+
+
+
+    def _engine_key(self, cfg: RedshiftConfig) -> str:
+        pwd_hash = hashlib.sha256(cfg.password.encode()).hexdigest()
+        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}|{pwd_hash}"
+    
+    def invalidate_pool(self):
+        key = self._engine_key(self._redshift_config)
+        with self.__class__._lock:
+            now = time.time()
+            last_time = self._last_dispose_time.get(key, 0)
+            # Throttle (5 seconds)
+            if now - last_time < 5:
+                return
+            engine = self._engines.get(key)
+            if engine:
+                logger.warning(f"Disposing engine pool for {key} due to stale OID")
+                engine.dispose()
+                self._last_dispose_time[key] = now
+
+    def _get_engine(self) -> Engine:
+        cfg = self._redshift_config
+        key = self._engine_key(cfg)
+
+        if key not in self._engines:
+            with self.__class__._lock: # thread-safe
+                if key not in self._engines: # recheck
+                    url = (
+                        f"redshift+redshift_connector://{cfg.username}:{cfg.password}"
+                        f"@{cfg.host}:{cfg.port}/{cfg.database}"
+                    )
+                    self._engines[key] = create_engine(
+                        url,
+                        pool_size=10,
+                        max_overflow=5,
+                        pool_timeout=30,
+                        pool_recycle=1800,
+                        pool_pre_ping=True,
+                        isolation_level="AUTOCOMMIT",
+                    )
+                    logger.info(f"Created new engine for {cfg.host}:{cfg.port}/{cfg.database}")
+
+        return self._engines[key]
+
+    def _is_maintenance_error(self, exc: Exception) -> bool:
+        """Return True when the error looks like a Redshift cluster reboot or maintenance."""
+        err = exc
+        while hasattr(err, "__cause__") and err.__cause__:
+            err = err.__cause__
+
+        if isinstance(err, ConnectionRefusedError):
+            return err.errno in (errno.ECONNREFUSED, 10061)
+
+        if isinstance(err, InterfaceError) and "ConnectionRefusedError" in str(err):
+            return True
+
+        return False
+
+    @contextmanager
+    def get_connection(self, retries: int = 3, delay: int = 5, query_timeout: int = 30):
+        """Context manager for database connections with retry mechanism"""
+        engine = self._get_engine()
+        last_exception = None
+
+        for attempt in range(retries):
+            conn = None
+            try:
+                conn = engine.connect()
+                conn.execute(text("SET enable_case_sensitive_identifier TO on"))
+                conn.execute(text(f"SET statement_timeout = {query_timeout * 1000}"))
+                if self.org_schema and self.set_default_schema:
+                    conn.execute(text(f"SET search_path TO {self.org_schema}"))
+                    logger.info(f"Search path set to: {self.org_schema}")
+
+                yield conn
+                return
+
+            except (OperationalError, InterfaceError, ProgrammingError) as e:
+                last_exception = e
+                if conn is not None:
+                    try:
+                        conn.rollback()
+                    except:
+                        pass
+                    conn.close()
+                    conn = None
+
+                if self._is_maintenance_error(e):
+                    logger.warning(f"Redshift maintenance detected (attempt {attempt + 1}/{retries}): {e}")
+                else:
+                    logger.warning(f"Connection failed (attempt {attempt + 1}/{retries}): {e}")
+
+                if attempt < retries - 1:
+                    time.sleep(delay)
+                    delay += 5
+                    continue
+
+                break
+
+            finally:
+                if conn is not None:
+                    conn.close()
+
+        raise BadRequest(
+            f"Failed to connect to Redshift after {retries} attempts : {str(last_exception)}"
+        ) from last_exception
+
+    def close_connection(self):
+        """Dispose all pooled engines managed by this class."""
+        for engine in self._engines.values():
+            engine.dispose()
+        self._engines.clear()
+        logger.info("All Redshift engine pools disposed.")
+
+    def is_valid_connection(self) -> bool:
+        """Pool health is managed by pool_pre_ping; no manual check needed."""
+        return True
+
+    def test_connection(self) -> bool:
+        """Verify the connection is reachable. Raises ConnectionError on failure."""
+        try:
+            with self.get_connection():
+                return True
+        except Exception as e:
+            error_msg = str(e)
+            if "Name or service not known" in error_msg:
+                user_msg = "Host not found. Please check the host address."
+            elif "Connection refused" in error_msg:
+                user_msg = "Connection refused. Please check the host and port."
+            elif "database" in error_msg.lower() and "does not exist" in error_msg:
+                user_msg = "Database does not exist. Please check the database name."
+            elif "password authentication failed" in error_msg.lower():
+                user_msg = "Authentication failed. Please check your username and password."
+            else:
+                user_msg = f"Redshift connection test failed: {error_msg}"
+            
+            raise ConnectionError(user_msg) from e

{fdc_shared_kernel-0.0.185 → fdc_shared_kernel-0.0.187}/shared_kernel/data_warehouse_handlers/snowflake_handler.py

@@ -30,7 +30,8 @@ class SnowflakeWarehouseHandler(WarehouseHandler):
             account=str(self.payload.get("account")).strip(),
             database=str(self.payload.get("database")).strip(),
             schema=str(self.payload.get("schema")).strip() if self.payload.get("schema") else None,
-            schema_list=self.payload.get("schema_list")
+            schema_list=self.payload.get("schema_list"),
+            pagination=self.pagination,
         )
 
     def _build_connection(self) -> SnowflakeWarehouseConnection:

fdc_shared_kernel-0.0.185/shared_kernel/data_warehouse_handlers/connections/redshift_warehouse_connection.py

@@ -1,178 +0,0 @@
-import errno
-import hashlib
-import time
-import threading
-
-from contextlib import contextmanager
-from typing import Dict
-
-from sqlalchemy import create_engine, text
-from sqlalchemy.engine import Engine
-from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError
-
-from shared_kernel.dataclasses.warehouse_configs import RedshiftConfig
-from shared_kernel.exceptions.http_exceptions import BadRequest
-from shared_kernel.interfaces.warehouse_connection import DataWarehouseConnection
-from shared_kernel.logger import Logger
-from shared_kernel.config import Config
-
-config = Config()
-logger = Logger(config.get("APP_NAME"))
-
-
-class RedshiftWarehouseConnection(DataWarehouseConnection):
-
-    _engines: Dict[str, Engine] = {}
-    _lock = threading.Lock()
-    _last_dispose_time: Dict[str, float] = {}
-
-    def __init__(
-        self,
-        source_config: RedshiftConfig,
-        set_default_schema: bool = True,
-    ):
-        if source_config is None:
-            raise ValueError(
-                "Missing required parameter: source_config (RedshiftConfig)"
-            )
-
-        super().__init__(source_config)
-        self._redshift_config = source_config
-        self.org_schema = source_config.schema if source_config.schema else None
-        self.set_default_schema = set_default_schema
-
-    def _engine_key(self, cfg: RedshiftConfig) -> str:
-        pwd_hash = hashlib.sha256(cfg.password.encode()).hexdigest()
-        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}|{pwd_hash}"
-    
-    def invalidate_pool(self):
-        key = self._engine_key(self._redshift_config)
-        with self.__class__._lock:
-            now = time.time()
-            last_time = self._last_dispose_time.get(key, 0)
-            # Throttle (5 seconds)
-            if now - last_time < 5:
-                return
-            engine = self._engines.get(key)
-            if engine:
-                logger.warning(f"Disposing engine pool for {key} due to stale OID")
-                engine.dispose()
-                self._last_dispose_time[key] = now
-
-    def _get_engine(self) -> Engine:
-        cfg = self._redshift_config
-        key = self._engine_key(cfg)
-
-        if key not in self._engines:
-            with self.__class__._lock: # thread-safe
-                if key not in self._engines: # recheck
-                    url = (
-                        f"redshift+redshift_connector://{cfg.username}:{cfg.password}"
-                        f"@{cfg.host}:{cfg.port}/{cfg.database}"
-                    )
-                    self._engines[key] = create_engine(
-                        url,
-                        pool_size=10,
-                        max_overflow=5,
-                        pool_timeout=30,
-                        pool_recycle=1800,
-                        pool_pre_ping=True,
-                        isolation_level="AUTOCOMMIT",
-                    )
-                    logger.info(f"Created new engine for {cfg.host}:{cfg.port}/{cfg.database}")
-
-        return self._engines[key]
-
-    def _is_maintenance_error(self, exc: Exception) -> bool:
-        """Return True when the error looks like a Redshift cluster reboot or maintenance."""
-        err = exc
-        while hasattr(err, "__cause__") and err.__cause__:
-            err = err.__cause__
-
-        if isinstance(err, ConnectionRefusedError):
-            return err.errno in (errno.ECONNREFUSED, 10061)
-
-        if isinstance(err, InterfaceError) and "ConnectionRefusedError" in str(err):
-            return True
-
-        return False
-
-    @contextmanager
-    def get_connection(self, retries: int = 3, delay: int = 5, query_timeout: int = 30):
-        """Context manager for database connections with retry mechanism"""
-        engine = self._get_engine()
-        last_exception = None
-
-        for attempt in range(retries):
-            conn = None
-            try:
-                conn = engine.connect()
-                conn.execute(text("SET enable_case_sensitive_identifier TO on"))
-                conn.execute(text(f"SET statement_timeout = {query_timeout * 1000}"))
-                if self.org_schema and self.set_default_schema:
-                    conn.execute(text(f"SET search_path TO {self.org_schema}"))
-                    logger.info(f"Search path set to: {self.org_schema}")
-
-                yield conn
-                return
-
-            except (OperationalError, InterfaceError, ProgrammingError) as e:
-                last_exception = e
-                if conn is not None:
-                    try:
-                        conn.rollback()
-                    except:
-                        pass
-                    conn.close()
-                    conn = None
-
-                if self._is_maintenance_error(e):
-                    logger.warning(f"Redshift maintenance detected (attempt {attempt + 1}/{retries}): {e}")
-                else:
-                    logger.warning(f"Connection failed (attempt {attempt + 1}/{retries}): {e}")
-
-                if attempt < retries - 1:
-                    time.sleep(delay)
-                    delay += 5
-                    continue
-
-                break
-
-            finally:
-                if conn is not None:
-                    conn.close()
-
-        raise BadRequest(
-            f"Failed to connect to Redshift after {retries} attempts : {str(last_exception)}"
-        ) from last_exception
-
-    def close_connection(self):
-        """Dispose all pooled engines managed by this class."""
-        for engine in self._engines.values():
-            engine.dispose()
-        self._engines.clear()
-        logger.info("All Redshift engine pools disposed.")
-
-    def is_valid_connection(self) -> bool:
-        """Pool health is managed by pool_pre_ping; no manual check needed."""
-        return True
-
-    def test_connection(self) -> bool:
-        """Verify the connection is reachable. Raises ConnectionError on failure."""
-        try:
-            with self.get_connection():
-                return True
-        except Exception as e:
-            error_msg = str(e)
-            if "Name or service not known" in error_msg:
-                user_msg = "Host not found. Please check the host address."
-            elif "Connection refused" in error_msg:
-                user_msg = "Connection refused. Please check the host and port."
-            elif "database" in error_msg.lower() and "does not exist" in error_msg:
-                user_msg = "Database does not exist. Please check the database name."
-            elif "password authentication failed" in error_msg.lower():
-                user_msg = "Authentication failed. Please check your username and password."
-            else:
-                user_msg = f"Redshift connection test failed: {error_msg}"
-            
-            raise ConnectionError(user_msg) from e