trovesuite 1.0.30__py3-none-any.whl → 1.0.32__py3-none-any.whl

trovesuite/configs/database.py

@@ -1,5 +1,20 @@
 """
 Database configuration and connection management
+
+FIXED VERSION - Removed pool reinitialization during runtime to prevent "unkeyed connection" errors.
+Key changes:
+- Pool created once at startup, never recreated during runtime
+- Removed connection validation queries that consume pool connections
+- Increased default pool size to 2 (safe for Azure Basic tier)
+- Simplified get_db_connection() - let pool.getconn() block naturally
+- Removed _recover_connection_pool() and _is_pool_valid() runtime checks
+
+NOTE: Database initialization is NOT automatic for package version.
+You must call initialize_database() explicitly in your application startup.
+Example in FastAPI:
+    @app.on_event("startup")
+    async def startup_event():
+        initialize_database()
 """
 from contextlib import contextmanager
 from typing import Generator, Optional
@@ -12,7 +27,7 @@ from .logging import get_logger
 
 logger = get_logger("database")
 
-# Database connection pool
+# Database connection pool - created once at startup, never replaced during runtime
 _connection_pool: Optional[psycopg2.pool.ThreadedConnectionPool] = None
 _initialization_lock = threading.Lock()
 
@@ -24,13 +39,11 @@ class DatabaseConfig:
         import os
         self.settings = db_settings
         self.database_url = self.settings.database_url
-        # CRITICAL: Azure PostgreSQL B_Standard_B1ms has very limited connections (~50 max, practical limit ~30-40)
-        # Default pool size is set to 1 to avoid connection exhaustion with multiple workers/replicas
-        # Formula: connections = pool_size × workers × replicas
-        # Example: 1 pool × 4 workers × 2 replicas = 8 connections (safe)
-        # With old default (5): 5 × 4 × 2 = 40 connections (exceeds limit!)
+        # Safe pool size for Azure Basic tier (B_Standard_B1ms)
+        # With 2 workers and pool_size=2: 2 × 2 = 4 connections (very safe)
+        # With 4 workers and pool_size=2: 4 × 2 = 8 connections (still safe, well under 50 limit)
         # Can be overridden with DB_POOL_SIZE environment variable
-        self.pool_size = int(os.getenv("DB_POOL_SIZE", "1"))
+        self.pool_size = int(os.getenv("DB_POOL_SIZE", "2"))
 
     def get_connection_params(self) -> dict:
         """Get database connection parameters"""
@@ -63,26 +76,12 @@ class DatabaseConfig:
         }
 
     def create_connection_pool(self) -> psycopg2.pool.ThreadedConnectionPool:
-        """Create a connection pool for psycopg2
-        
-        Note: For Azure PostgreSQL B_Standard_B1ms, keep pool_size ≤ 1 per worker
-        to avoid connection exhaustion. Consider using PgBouncer for higher concurrency.
-        """
+        """Create a connection pool for psycopg2"""
         try:
-            # For Azure PostgreSQL, use minimum pool size to avoid connection exhaustion
-            # Multiple replicas/workers can quickly exhaust database connections on Basic tier
             import os
             dsn = os.getenv("DATABASE_URL", "") or str(self.database_url or "")
             is_azure = "database.azure.com" in dsn.lower()
-            
-            # Ensure pool size is appropriate for Azure Basic tier
             pool_size = self.pool_size
-            if is_azure and pool_size > 2:
-                logger.warning(
-                    f"⚠️  Pool size {pool_size} may be too high for Azure Basic tier. "
-                    f"Recommended: 1-2 connections per worker. "
-                    f"Set DB_POOL_SIZE=1 to avoid connection exhaustion."
-                )
             
             pool = psycopg2.pool.ThreadedConnectionPool(
                 minconn=1,
@@ -95,7 +94,6 @@ class DatabaseConfig:
             )
             return pool
         except psycopg2.OperationalError as e:
-            # Check if it's a connection limit error
             error_str = str(e).lower()
             if any(keyword in error_str for keyword in ["connection", "slot", "limit", "exhausted", "too many"]):
                 logger.error("⚠️  Database connection limit reached!")
@@ -105,8 +103,8 @@ class DatabaseConfig:
                 logger.error("   3. Too many Gunicorn workers (GUNICORN_WORKERS environment variable)")
                 logger.error("   4. Connections not being properly returned to pool")
                 logger.error("   Solutions:")
-                logger.error("   - Set DB_POOL_SIZE=1 (recommended for Azure Basic tier)")
-                logger.error("   - Reduce GUNICORN_WORKERS (default: 4)")
+                logger.error("   - Set DB_POOL_SIZE=2 (current default)")
+                logger.error("   - Reduce GUNICORN_WORKERS (default: 2)")
                 logger.error("   - Consider using PgBouncer for connection pooling")
                 logger.error("   - Upgrade to a higher PostgreSQL tier if needed")
             logger.error(f"Failed to create database connection pool: {str(e)}")
@@ -116,7 +114,7 @@ class DatabaseConfig:
             raise
 
     def test_connection(self) -> bool:
-        """Test database connection"""
+        """Test database connection (only used at startup)"""
         try:
             with psycopg2.connect(**self.get_connection_params()) as conn:
                 with conn.cursor() as cursor:
@@ -136,190 +134,104 @@ db_config = DatabaseConfig()
 
 
 def initialize_database():
-    """Initialize database connections and pool"""
+    """
+    Initialize database connections and pool.
+    This should ONLY be called at application startup.
+    Pool is created once and never recreated during runtime.
+    
+    NOTE: For package version, this must be called explicitly in application startup.
+    """
     global _connection_pool
     
-    # Close existing pool if it exists (cleanup before reinitializing)
-    if _connection_pool is not None:
-        try:
-            _connection_pool.closeall()
-            logger.info("Closed existing connection pool before reinitialization")
-        except Exception as e:
-            logger.warning(f"Error closing existing pool: {str(e)}")
-        _connection_pool = None
-
-    try:
-        # Test connection first
-        if not db_config.test_connection():
-            raise Exception("Database connection test failed")
-
-        # Create connection pool
-        _connection_pool = db_config.create_connection_pool()
+    with _initialization_lock:
+        # If pool already exists, don't recreate it
+        if _connection_pool is not None:
+            logger.warning("Database pool already initialized, skipping reinitialization")
+            return
         
-        # Verify pool was created successfully
-        if _connection_pool is None:
-            raise Exception("Connection pool creation returned None")
-
-        logger.info("✅ Database initialization completed successfully")
-
-    except Exception as e:
-        logger.error(f"❌ Database initialization failed: {str(e)}")
-        _connection_pool = None  # Ensure pool is None on failure
-        raise
-
+        try:
+            # Test connection first (only at startup)
+            if not db_config.test_connection():
+                raise Exception("Database connection test failed")
 
-def _is_pool_valid(pool) -> bool:
-    """Check if the connection pool is valid and usable"""
-    if pool is None:
-        return False
-    try:
-        # ThreadedConnectionPool doesn't expose a direct "closed" attribute
-        # Check if pool has the necessary internal structures
-        if not hasattr(pool, '_pool'):
-            return False
-        if pool._pool is None:
-            return False
-        # Additional check: verify pool has connection parameters
-        if not hasattr(pool, '_kwargs'):
-            return False
-        return True
-    except (AttributeError, Exception) as e:
-        logger.debug(f"Pool validation check: {str(e)}")
-        return False
+            # Create connection pool (only once at startup)
+            _connection_pool = db_config.create_connection_pool()
+            
+            # Verify pool was created successfully
+            if _connection_pool is None:
+                raise Exception("Connection pool creation returned None")
 
+            logger.info("✅ Database initialization completed successfully")
 
-def _recover_connection_pool() -> bool:
-    """Attempt to recover the connection pool with retry logic"""
-    global _connection_pool, _initialization_lock
-    import time
-    
-    with _initialization_lock:
-        # Double-check after acquiring lock
-        if _connection_pool is not None and _is_pool_valid(_connection_pool):
-            return True
-        
-        # Close invalid pool if it exists
-        if _connection_pool is not None:
-            try:
-                _connection_pool.closeall()
-                logger.info("Closed invalid connection pool")
-            except Exception as e:
-                logger.warning(f"Error closing invalid pool: {str(e)}")
-            _connection_pool = None
-        
-        # Retry with exponential backoff
-        max_retries = 3
-        base_delay = 1  # Start with 1 second
-        
-        for attempt in range(1, max_retries + 1):
-            try:
-                logger.warning(f"Attempting to reinitialize connection pool (attempt {attempt}/{max_retries})...")
-                initialize_database()
-                
-                if _connection_pool is not None and _is_pool_valid(_connection_pool):
-                    logger.info(f"✅ Connection pool reinitialized successfully (attempt {attempt})")
-                    return True
-                else:
-                    logger.warning(f"Pool initialized but validation failed (attempt {attempt})")
-                    
-            except Exception as e:
-                logger.error(f"Pool reinitialization attempt {attempt} failed: {str(e)}")
-                if attempt < max_retries:
-                    delay = base_delay * (2 ** (attempt - 1))  # Exponential backoff: 1s, 2s, 4s
-                    logger.info(f"Retrying in {delay} seconds...")
-                    time.sleep(delay)
-        
-        logger.error("❌ Failed to reinitialize connection pool after all retries")
-        return False
+        except Exception as e:
+            logger.error(f"❌ Database initialization failed: {str(e)}")
+            _connection_pool = None  # Ensure pool is None on failure
+            raise
 
 
 def get_connection_pool() -> psycopg2.pool.ThreadedConnectionPool:
-    """Get the database connection pool, with automatic reinitialization if needed"""
+    """
+    Get the database connection pool.
+    Pool must be initialized at startup. This function will raise if pool is None.
+    """
     global _connection_pool
     
-    # Fast path: pool exists and is valid
-    if _connection_pool is not None and _is_pool_valid(_connection_pool):
-        return _connection_pool
-    
-    # Pool is None or invalid, attempt recovery
-    if not _recover_connection_pool():
+    if _connection_pool is None:
         error_msg = (
-            "Database connection pool is unavailable. This usually means:\n"
-            "1. Database server is unreachable or down\n"
-            "2. Network connectivity issues\n"
-            "3. Database credentials are incorrect\n"
-            "4. Connection pool exhausted or closed\n"
-            "5. Database initialization failed\n"
-            "Please check the startup logs and database status."
+            "Database connection pool is not initialized. "
+            "Please ensure initialize_database() was called at application startup."
         )
         logger.error(error_msg)
         raise Exception(error_msg)
     
-    if _connection_pool is None:
-        error_msg = "Connection pool recovery completed but pool is still None"
-        logger.error(error_msg)
-        raise Exception(error_msg)
-    
     return _connection_pool
 
 
-def _validate_connection(conn) -> bool:
-    """Validate if a connection is still alive"""
-    try:
-        # Check if connection is closed first
-        if conn.closed:
-            return False
-        
-        # Test if connection is alive with a simple query
-        with conn.cursor() as cursor:
-            cursor.execute("SELECT 1")
-            cursor.fetchone()
-        return True
-    except (psycopg2.OperationalError, psycopg2.InterfaceError, psycopg2.DatabaseError) as e:
-        logger.warning(f"Connection validation failed: {str(e)}")
-        return False
-    except Exception as e:
-        logger.warning(f"Unexpected error during connection validation: {str(e)}")
-        return False
-
-
 @contextmanager
 def get_db_connection():
-    """Get a database connection from the pool (context manager)"""
+    """
+    Get a database connection from the pool (context manager).
+    
+    This is simplified - we let pool.getconn() block naturally.
+    No retries, no validation queries, no pool recovery.
+    """
     pool = get_connection_pool()
     conn = None
     try:
+        # Get connection from pool - this will block if pool is exhausted
+        # That's the correct behavior - let backpressure happen naturally
         conn = pool.getconn()
-
-        # Validate connection before using it
-        if not _validate_connection(conn):
-            logger.warning("Stale connection detected, getting new connection")
-            pool.putconn(conn, close=True)
-            conn = pool.getconn()
-
         logger.debug("Database connection acquired from pool")
         yield conn
     except Exception as e:
-        logger.error(f"Database connection error: {str(e)}")
-        if conn:
+        # If connection exists and isn't closed, rollback transaction
+        if conn and not conn.closed:
             try:
-                # Only rollback if connection is still open
-                if not conn.closed:
-                    conn.rollback()
-            except (psycopg2.OperationalError, psycopg2.InterfaceError) as rollback_error:
-                logger.warning(f"Could not rollback closed connection: {str(rollback_error)}")
+                conn.rollback()
+            except Exception as rollback_error:
+                logger.warning(f"Could not rollback transaction: {str(rollback_error)}")
+        # Re-raise the exception - don't retry here
         raise
     finally:
+        # Always return connection to pool
         if conn:
             try:
-                # If connection is broken, close it instead of returning to pool
                 if conn.closed:
+                    # If connection is closed, tell pool to close it instead of returning
                     pool.putconn(conn, close=True)
                 else:
+                    # Return connection to pool normally
                     pool.putconn(conn)
                 logger.debug("Database connection returned to pool")
             except Exception as put_error:
+                # Log error but don't fail - connection will be cleaned up by pool
                 logger.error(f"Error returning connection to pool: {str(put_error)}")
+                # Try to close connection manually as last resort
+                try:
+                    if not conn.closed:
+                        conn.close()
+                except Exception:
+                    pass
 
 
 @contextmanager
@@ -392,6 +304,7 @@ class DatabaseManager:
                 cursor.execute("INSERT INTO table2 ...")
                 # Auto-commits on success, auto-rollbacks on exception
         """
+        # Use get_db_connection() instead of directly accessing pool
         with get_db_connection() as conn:
             cursor = conn.cursor(cursor_factory=RealDictCursor)
             try:
@@ -415,7 +328,10 @@ class DatabaseManager:
 
     @staticmethod
     def health_check() -> dict:
-        """Perform database health check"""
+        """
+        Perform database health check.
+        Health checks are allowed to fail - they don't attempt to repair the pool.
+        """
         try:
             with get_db_cursor() as cursor:
                 cursor.execute("SELECT version(), current_database(), current_user")
@@ -451,9 +367,9 @@ class DatabaseManager:
             }
 
 
-# NOTE: Database initialization is NOT automatic
+# NOTE: Database initialization is NOT automatic for package version
 # You must call initialize_database() explicitly in your application startup
 # Example in FastAPI:
 #   @app.on_event("startup")
 #   async def startup_event():
-#       initialize_database()
+#       initialize_database()

{trovesuite-1.0.30.dist-info → trovesuite-1.0.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trovesuite
-Version: 1.0.30
+Version: 1.0.32
 Summary: TroveSuite services package providing authentication, authorization, notifications, Azure Storage, and other enterprise services for TroveSuite applications
 Home-page: https://dev.azure.com/brightgclt/trovesuite/_git/packages
 Author: Bright Debrah Owusu

{trovesuite-1.0.30.dist-info → trovesuite-1.0.32.dist-info}/RECORD

@@ -6,7 +6,7 @@ trovesuite/auth/auth_read_dto.py,sha256=e27JqKVPVUM83A_mYF452QCflsvGNo7aKje7q_ur
 trovesuite/auth/auth_service.py,sha256=TQOJFG0AzhPGwZBAXVxMkHxyG2wyct4Zcoq4z0cVBO4,22201
 trovesuite/auth/auth_write_dto.py,sha256=rdwI7w6-9QZGv1H0PAGrjkLBCzaMHjgPIXeLb9RmNec,234
 trovesuite/configs/__init__.py,sha256=h1mSZOaZ3kUy1ZMO_m9O9KklsxywM0RfMVZLh9h9WvQ,328
-trovesuite/configs/database.py,sha256=x6ucbF7lNLSNqeCqFgJ4NUbkkAvYifdcQbOd9kxZUPQ,18414
+trovesuite/configs/database.py,sha256=u4aTAZD4vyByzwPkmH47rAZ8iA4nsYyMiaL634uUcAM,15008
 trovesuite/configs/logging.py,sha256=mGjR2d4urVNry9l5_aXycMMtcY2RAFIpEL35hw33KZg,9308
 trovesuite/configs/settings.py,sha256=s-RvzFI7IR8-Q_A0ZRxlTr-iDoY6MvbxM0RkbhmJOL4,6915
 trovesuite/entities/__init__.py,sha256=Dbl_03Bueyh2vOP2hykd40MmNMrl5nNHSRGP-kqwwNo,160
@@ -27,8 +27,8 @@ trovesuite/storage/storage_write_dto.py,sha256=vl1iCZ93bpFmpvkCrn587QtMtOA_TPDse
 trovesuite/utils/__init__.py,sha256=mDZuY77BphvQFYLmcWxjP5Tcq9ZZ3WXJWBKB1v6wzHU,185
 trovesuite/utils/helper.py,sha256=qpd-EWPaX3-QJA5xvxb4s9rEb9W2RKPCDXcdAKSUwSM,30858
 trovesuite/utils/templates.py,sha256=_92k4-EkqWs-h0LNJxPgorbspmp24kDngS7O3qWIFyQ,20388
-trovesuite-1.0.30.dist-info/licenses/LICENSE,sha256=EJT35ct-Q794JYPdAQy3XNczQGKkU1HzToLeK1YVw2s,1070
-trovesuite-1.0.30.dist-info/METADATA,sha256=jlg46ILbbTBMpJ7SMUVcH9lBjA_HmnFm6oejaTshjX4,21737
-trovesuite-1.0.30.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-trovesuite-1.0.30.dist-info/top_level.txt,sha256=GzKhG_-MTaxeHrIgkGkBH_nof2vroGFBrjeHKWUIwNc,11
-trovesuite-1.0.30.dist-info/RECORD,,
+trovesuite-1.0.32.dist-info/licenses/LICENSE,sha256=EJT35ct-Q794JYPdAQy3XNczQGKkU1HzToLeK1YVw2s,1070
+trovesuite-1.0.32.dist-info/METADATA,sha256=HoifrGTHx071bljGeM7PCOeg4taR1Anie_s5SDwaOCs,21737
+trovesuite-1.0.32.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+trovesuite-1.0.32.dist-info/top_level.txt,sha256=GzKhG_-MTaxeHrIgkGkBH_nof2vroGFBrjeHKWUIwNc,11
+trovesuite-1.0.32.dist-info/RECORD,,