fdc-shared-kernel 0.0.170__tar.gz → 0.0.172__tar.gz

{fdc_shared_kernel-0.0.170 → fdc_shared_kernel-0.0.172}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.170
+Version: 0.0.172
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.170 → fdc_shared_kernel-0.0.172}/fdc_shared_kernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.170
+Version: 0.0.172
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.170 → fdc_shared_kernel-0.0.172}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fdc_shared_kernel"
-version = "0.0.170"
+version = "0.0.172"
 requires-python = ">=3.7"
 readme = "README_pypi.md"
 description = "Shared library for microservice"

{fdc_shared_kernel-0.0.170 → fdc_shared_kernel-0.0.172}/shared_kernel/data_warehouse_handlers/connections/mssql_warehouse_connection.py

@@ -1,3 +1,4 @@
+import hashlib
 import time
 import threading
 
@@ -44,7 +45,8 @@ class MSSQLWarehouseConnection(DataWarehouseConnection):
         self.search = pagination.get("search") if isinstance(pagination, dict) else None
 
     def _engine_key(self, cfg: MSSQLConfig) -> str:
-        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}"
+        pwd_hash = hashlib.sha256(cfg.password.encode()).hexdigest()
+        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}|{pwd_hash}"
 
     def _get_engine(self) -> Engine:
         cfg = self._mssql_config

{fdc_shared_kernel-0.0.170 → fdc_shared_kernel-0.0.172}/shared_kernel/data_warehouse_handlers/connections/mysql_warehouse_connection.py

@@ -1,3 +1,4 @@
+import hashlib
 import time
 import threading
 import pymysql
@@ -53,7 +54,8 @@ class MySQLWarehouseConnection(DataWarehouseConnection):
         self.search = pagination.get("search") if isinstance(pagination, dict) else None
 
     def _engine_key(self, cfg: MySQLConfig) -> str:
-        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}"
+        pwd_hash = hashlib.sha256(cfg.password.encode()).hexdigest()
+        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}|{pwd_hash}"
 
     def _get_engine(self) -> Engine:
         cfg = self._mysql_config

{fdc_shared_kernel-0.0.170 → fdc_shared_kernel-0.0.172}/shared_kernel/data_warehouse_handlers/connections/postgresql_warehouse_connection.py

@@ -1,3 +1,4 @@
+import hashlib
 import time
 import threading
 
@@ -45,7 +46,8 @@ class PostgreSQLWarehouseConnection(DataWarehouseConnection):
         self.search = pagination.get("search") if isinstance(pagination, dict) else None
 
     def _engine_key(self, cfg: PostgreSQLConfig) -> str:
-        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}"
+        pwd_hash = hashlib.sha256(cfg.password.encode()).hexdigest()
+        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}|{pwd_hash}"
 
     def _get_engine(self) -> Engine:
         cfg = self._postgres_config

{fdc_shared_kernel-0.0.170 → fdc_shared_kernel-0.0.172}/shared_kernel/data_warehouse_handlers/connections/redshift_warehouse_connection.py

@@ -1,4 +1,5 @@
 import errno
+import hashlib
 import time
 import threading
 
@@ -23,6 +24,7 @@ class RedshiftWarehouseConnection(DataWarehouseConnection):
 
     _engines: Dict[str, Engine] = {}
     _lock = threading.Lock()
+    _last_dispose_time: Dict[str, float] = {}
 
     def __init__(
         self,
@@ -40,7 +42,22 @@ class RedshiftWarehouseConnection(DataWarehouseConnection):
         self.set_default_schema = set_default_schema
 
     def _engine_key(self, cfg: RedshiftConfig) -> str:
-        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}"
+        pwd_hash = hashlib.sha256(cfg.password.encode()).hexdigest()
+        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}|{pwd_hash}"
+    
+    def invalidate_pool(self):
+        key = self._engine_key(self._redshift_config)
+        with self.__class__._lock:
+            now = time.time()
+            last_time = self._last_dispose_time.get(key, 0)
+            # Throttle (5 seconds)
+            if now - last_time < 5:
+                return
+            engine = self._engines.get(key)
+            if engine:
+                logger.warning(f"Disposing engine pool for {key} due to stale OID")
+                engine.dispose(close=False)
+                self._last_dispose_time[key] = now
 
     def _get_engine(self) -> Engine:
         cfg = self._redshift_config

{fdc_shared_kernel-0.0.170 → fdc_shared_kernel-0.0.172}/shared_kernel/data_warehouse_handlers/query_executors/redshift_query_executor.py

@@ -25,6 +25,9 @@ class RedshiftQueryExecutor(DataWarehouseQueryExecutor):
     def __init__(self, warehouse_connection: DataWarehouseConnection) -> None:
         self.warehouse_connection = warehouse_connection
 
+    def invalidate_pool(self):
+        self.warehouse_connection.invalidate_pool()
+
     def execute_query(
         self,
         query: str,
@@ -56,13 +59,37 @@ class RedshiftQueryExecutor(DataWarehouseQueryExecutor):
                     return []
 
             except (OperationalError, ProgrammingError, InterfaceError) as e:
+                error_msg = str(e)
+                if "could not open relation with OID" in error_msg:
+                    logger.warning("Detected stale OID. Invalidating connection pool and retrying...")
+                    self.invalidate_pool()
+                    try:
+                        with self.warehouse_connection.get_connection() as conn:
+                            result = (
+                                conn.execute(text(query), params)
+                                if params
+                                else conn.execute(text(query))
+                            )
+                            # Check if the query returns rows
+                            if result.returns_rows:
+                                columns = result.keys()
+                                return [dict(zip(columns, row)) for row in result.fetchall()]
+                            else:
+                                # Return an empty list for DDL queries
+                                return []
+                    except Exception as retry_error:
+                        logger.error(
+                            f"Retry after OID failure also failed: {retry_error}",
+                            exc_info=True
+                        )
+                        raise Exception(
+                            f"Query failed even after retry: {extract_error_message(str(retry_error))}"
+                        ) from retry_error
                 logger.error(f"Query failed: {query} — {e}", exc_info=True)
-                user_msg = extract_error_message(str(e))
-                raise Exception(f"Query execution failed: {user_msg}") from e
+                raise Exception(f"Query execution failed: {extract_error_message(error_msg)}") from e
             except Exception as e:
-                user_msg = extract_error_message(str(e))
-                raise Exception(f"Query execution failed: {user_msg}") from e
-            
+                raise Exception(f"Query execution failed: {extract_error_message(str(e))}") from e
+
     def execute_queries_sequential(
         self,
         queries: List[str],
@@ -98,12 +125,14 @@ class RedshiftQueryExecutor(DataWarehouseQueryExecutor):
                         rows = []
                     results.append(rows)
                 except (OperationalError, ProgrammingError, InterfaceError) as e:
+                    if "could not open relation with OID" in str(e):
+                        logger.warning("OID error during sequential execution. Invalidating pool and aborting batch.")
+                        self.invalidate_pool()
+                        raise Exception("Sequential execution aborted due to stale OID. Retry entire batch.")
                     logger.error(f"Sequential query failed: {query} — {e}", exc_info=True)
-                    user_msg = extract_error_message(str(e))
-                    raise Exception(f"Sequential query execution failed: {user_msg}") from e
+                    raise Exception(f"Sequential query execution failed: {extract_error_message(str(e))}") from e
                 except Exception as e:
-                    user_msg = extract_error_message(str(e))
-                    raise Exception(f"Sequential query execution failed: {user_msg}") from e
+                    raise Exception(f"Sequential query execution failed: {extract_error_message(str(e))}") from e
             return results
 
     def execute_queries(
@@ -180,10 +209,25 @@ class RedshiftQueryExecutor(DataWarehouseQueryExecutor):
                     return [], []
 
             except (OperationalError, ProgrammingError, InterfaceError) as e:
+                error_msg = str(e)
+                if "could not open relation with OID" in error_msg:
+                    logger.warning("Detected stale OID (column types). Invalidating pool and retrying...")
+                    self.invalidate_pool()
+                    try:
+                        with self.warehouse_connection.get_connection() as conn:
+                            result = (
+                                conn.execute(text(query), params)
+                                if params
+                                else conn.execute(text(query))
+                            )
+                            description = result.cursor.description
+                            columns = result.keys()
+                            rows = [dict(zip(columns, row)) for row in result.fetchall()]
+                            return rows, description
+                    except Exception as retry_error:
+                        logger.error(f"Retry after OID failure also failed: {retry_error}",exc_info=True)
+                        raise Exception(f"Query failed even after retry: {extract_error_message(str(retry_error))}") from retry_error
                 logger.error(f"Query failed: {query} — {e}", exc_info=True)
-                user_msg = extract_error_message(str(e))
-                raise Exception(f"Query execution failed: {user_msg}") from e
+                raise Exception(f"Query execution failed: {extract_error_message(error_msg)}") from e
             except Exception as e:
-                user_msg = extract_error_message(str(e))
-                raise Exception(f"Query execution failed: {user_msg}") from e
-            
+                raise Exception(f"Query execution failed: {extract_error_message(str(e))}") from e