fdc-shared-kernel 0.0.191__tar.gz → 0.0.193__tar.gz

{fdc_shared_kernel-0.0.191 → fdc_shared_kernel-0.0.193}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.191
+Version: 0.0.193
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.191 → fdc_shared_kernel-0.0.193}/fdc_shared_kernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.191
+Version: 0.0.193
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.191 → fdc_shared_kernel-0.0.193}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fdc_shared_kernel"
-version = "0.0.191"
+version = "0.0.193"
 requires-python = ">=3.7"
 readme = "README_pypi.md"
 description = "Shared library for microservice"

fdc_shared_kernel-0.0.193/shared_kernel/data_warehouse_handlers/connections/fabric_sql_warehouse_connection.py

@@ -0,0 +1,256 @@
+import hashlib
+import re
+import struct
+import time
+import threading
+
+from contextlib import contextmanager
+from typing import Dict
+
+from azure.identity import ClientSecretCredential
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError
+
+from shared_kernel.dataclasses.warehouse_configs import FabricSQLConfig
+from shared_kernel.exceptions.http_exceptions import BadRequest
+from shared_kernel.interfaces.warehouse_connection import DataWarehouseConnection
+from shared_kernel.data_warehouse_handlers.utils import extract_sql_server_error_message
+from shared_kernel.logger import Logger
+from shared_kernel.config import Config
+
+config = Config()
+logger = Logger(config.get("APP_NAME"))
+
+
+class FabricSQLWarehouseConnection(DataWarehouseConnection):
+
+    _engines: Dict[str, Engine] = {}
+    _tokens: Dict[str, dict] = {}
+    _lock = threading.Lock()
+
+    def __init__(
+        self,
+        source_config: FabricSQLConfig,
+        set_default_schema: bool = True,
+    ):
+        if source_config is None:
+            raise ValueError("Missing required parameter: source_config (FabricSQLConfig)")
+
+        self._fabric_config = source_config
+        self.org_schema = source_config.schema if source_config.schema else None
+        self.set_default_schema = set_default_schema
+
+        pagination = getattr(source_config, "pagination", None) or {}
+        self.page = pagination.get("page") if isinstance(pagination, dict) else None
+        self.limit = pagination.get("limit") if isinstance(pagination, dict) else None
+        self.search = pagination.get("search") if isinstance(pagination, dict) else None
+
+    def _engine_key(self, cfg: FabricSQLConfig) -> str:
+        # All five credential fields are part of the key so that any change to any of them
+        # forces a new engine + fresh token request (no stale-cache bypass).
+        # Password is SHA-256 hashed so the raw secret is never stored in a dict key.
+        pwd_hash = hashlib.sha256(cfg.password.encode()).hexdigest()
+        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}|{cfg.tenant_id}|{pwd_hash}"
+
+    def _get_fabric_connection_params(self, cfg: FabricSQLConfig) -> str:
+        """
+        Builds the base connection parameters for Microsoft Fabric SQL.
+        
+        Returns:
+            str: Connection string
+        """
+        import urllib.parse
+        # Build ODBC params. We omit UID and Password fields entirely from the connection string.
+        # Authentication is handled solely by injecting the access token into the connection handle.
+        odbc_params = urllib.parse.quote_plus(
+            f"DRIVER={{ODBC Driver 18 for SQL Server}};"
+            f"SERVER={cfg.host},{int(cfg.port)};"
+            f"DATABASE={cfg.database};"
+            "Encrypt=yes;"
+            "TrustServerCertificate=no;"
+        )
+        # We use odbc_connect query param to prevent SQLAlchemy dialect from auto-injecting
+        # conflicting credentials/parameters (avoiding error FA005).
+        connection_string = f"mssql+pyodbc:///?odbc_connect={odbc_params}"
+        return connection_string
+
+    def _get_engine(self) -> Engine:
+        cfg = self._fabric_config
+        key = self._engine_key(cfg)
+
+        if key not in self._engines:
+            with self.__class__._lock: # thread-safe
+                if key not in self._engines: # recheck
+                    from sqlalchemy import event
+                    
+                    connection_string = self._get_fabric_connection_params(cfg)
+
+                    engine = create_engine(
+                        connection_string,
+                        pool_size=10,
+                        max_overflow=5,
+                        pool_timeout=30,
+                        pool_recycle=1800,
+                        pool_pre_ping=True,
+                    )
+                    
+                    # Ensure a thread-safe token cache structure exists for this database key.
+                    # Each unique database configuration gets its own token cache, expiration tracker, and lock.
+                    if key not in self._tokens:
+                        self._tokens[key] = {
+                            "token": None,       # Stores the packed binary token structure
+                            "expires_at": 0,     # Unix timestamp indicating token expiration time
+                            "lock": threading.Lock() # Lock to synchronize concurrent token refreshes
+                        }
+                    
+                    # Register an event listener on the engine to intercept connection establishment.
+                    # Every time SQLAlchemy checks out or opens a new raw DBAPI connection from the pool,
+                    # this listener executes to inject a valid, fresh Entra ID access token.
+                    @event.listens_for(engine, "do_connect")
+                    def provide_token(dialect, conn_rec, cargs, cparams):
+                        now = time.time()
+                        token_info = self._tokens[key]
+                        
+                        # Check if the token is missing or near expiration (within 5 minutes / 300 seconds).
+                        # This buffer ensures we don't send a token that is about to expire.
+                        if token_info["token"] is None or now >= token_info["expires_at"] - 300:
+                            # Acquire the database-specific lock to serialize token generation.
+                            # If 10 threads try to connect concurrently when a token expires,
+                            # only the first thread executes the token fetch; others block here.
+                            with token_info["lock"]:
+                                # Double-checked locking pattern: once the lock is acquired,
+                                # re-verify if another thread updated the token while we were waiting.
+                                if token_info["token"] is None or now >= token_info["expires_at"] - 300:
+                                    tenant_id = getattr(cfg, "tenant_id", None)
+                                    if not tenant_id:
+                                        raise BadRequest(
+                                            "tenant_id is required in FabricSQLConfig for authenticating "
+                                            "to Microsoft Fabric via Active Directory Service Principal."
+                                        )
+
+                                    # Validate that tenant_id looks like a UUID or a domain name.
+                                    # A truncated/malformed UUID (e.g. 'a815263-...' instead of '1a815263-...')
+                                    # causes a cryptic MSAL authority error — catch it here with a clear message.
+                                    _TENANT_UUID = re.compile(
+                                        r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
+                                    )
+                                    _TENANT_DOMAIN = re.compile(r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
+                                    if not (_TENANT_UUID.match(tenant_id) or _TENANT_DOMAIN.match(tenant_id)):
+                                        raise BadRequest(
+                                            f"Invalid tenant_id '{tenant_id}': must be a full UUID "
+                                            f"(e.g. '1a815263-e4f7-4552-b141-ce31ce582f69') or a domain "
+                                            f"(e.g. 'contoso.onmicrosoft.com'). "
+                                            f"Check the stored tenant_id — a common mistake is a truncated UUID "
+                                            f"(missing leading characters)."
+                                        )
+
+                                    # --- Secure diagnostic logging (never logs the full secret) ---
+                                    _secret = cfg.password or ""
+                                    _masked = (
+                                        f"{_secret[:3]}...{_secret[-3:]}" if len(_secret) > 6 else "***"
+                                    )
+                                    logger.info(
+                                        f"Fetching Entra ID token | tenant_id={tenant_id} "
+                                        f"| client_id={cfg.username} "
+                                        f"| secret_len={len(_secret)} "
+                                        f"| secret_preview={_masked}"
+                                    )
+
+                                    try:
+                                        cred = ClientSecretCredential(
+                                            tenant_id=tenant_id,
+                                            client_id=cfg.username,
+                                            client_secret=cfg.password
+                                        )
+                                        token_obj = cred.get_token("https://database.windows.net/.default")
+                                        logger.info(f"Entra ID token acquired via tenant_id={tenant_id}")
+                                    except Exception as auth_err:
+                                        raise BadRequest(
+                                            f"Authentication failed for tenant_id: {auth_err}. "
+                                            "Verify that tenant_id, client_id (user), and client_secret (password) "
+                                            "are all correct and the App Registration has not expired."
+                                        ) from auth_err
+
+                                    # Encode the OAuth token to UTF-16-LE bytes.
+                                    # Pack the payload into the binary structure required by ODBC Driver 18:
+                                    # [4 bytes representing the length of the token string] followed by [the token string bytes].
+                                    token_bytes = token_obj.token.encode("utf-16-le")
+                                    token_info["token"] = struct.pack(f"<I{len(token_bytes)}s", len(token_bytes), token_bytes)
+                                    token_info["expires_at"] = token_obj.expires_on
+                                    logger.info("Successfully fetched and cached fresh Entra ID access token.")
+                                    
+                        # Inject the packed binary token structure into pyodbc's attrs_before connection attribute list.
+                        # Attribute ID 1256 maps to SQL_COPT_SS_ACCESS_TOKEN.
+                        cparams["attrs_before"] = {1256: token_info["token"]}
+
+                    self._engines[key] = engine
+                    logger.info(
+                        f"Created new Fabric SQL engine for {cfg.host}:{cfg.port}/{cfg.database} with dynamic token provider"
+                    )
+
+        return self._engines[key]
+
+    @contextmanager
+    def get_connection(self, retries: int = 3, delay: int = 5, query_timeout: int = 30):
+        """Context manager for database connections with retry mechanism"""
+        engine = self._get_engine()
+        last_exception = None
+
+        for attempt in range(retries):
+            conn = None
+            try:
+                conn = engine.connect()
+                yield conn
+                return
+
+            except (OperationalError, InterfaceError, ProgrammingError) as e:
+                last_exception = e
+                if conn is not None:
+                    conn.close()
+                    conn = None
+
+                error_message = extract_sql_server_error_message(str(e))
+                logger.warning(
+                    f"Fabric SQL connection failed (attempt {attempt + 1}/{retries}): {error_message}"
+                )
+
+                if attempt < retries - 1:
+                    time.sleep(delay)
+                    delay += 5
+                    continue
+
+                break
+
+            finally:
+                if conn is not None:
+                    conn.close()
+
+        raise BadRequest(
+            f"Failed to connect to Fabric SQL after {retries} attempts"
+        ) from last_exception
+
+    def close_connection(self):
+        """Dispose all pooled engines managed by this class."""
+        for engine in self._engines.values():
+            engine.dispose()
+        self._engines.clear()
+        self._tokens.clear()
+        logger.info("All Fabric SQL engine pools disposed.")
+
+    def is_valid_connection(self) -> bool:
+        """Pool health is managed by pool_pre_ping; no manual check needed."""
+        return True
+
+    def test_connection(self) -> bool:
+        """Verify the connection is reachable. Raises on failure."""
+        try:
+            with self.get_connection() as conn:
+                conn.execute(text("SELECT 1"))
+            return True
+        except OperationalError as e:
+            if "authentication failed" in str(e).lower():
+                raise ValueError("Invalid username or password")
+            raise ConnectionError(f"Database error: {str(e)}")
+        except InterfaceError:
+            raise ConnectionError("Cannot connect to database server")

{fdc_shared_kernel-0.0.191 → fdc_shared_kernel-0.0.193}/shared_kernel/data_warehouse_handlers/fabric_sql_handler.py

@@ -52,9 +52,6 @@ class FabricSQLWarehouseHandler(WarehouseHandler):
         connection = self._build_connection()
         return FabricSQLQueryExecutor(warehouse_connection=connection)
 
-    def get_constants_section(self) -> str:
-        return "mssql"
-
     def get_query_object(self) -> MSSQLQuery:
         return MSSQLQuery
 

fdc_shared_kernel-0.0.191/shared_kernel/data_warehouse_handlers/connections/fabric_sql_warehouse_connection.py

@@ -1,183 +0,0 @@
-import hashlib
-import time
-import threading
-
-from contextlib import contextmanager
-from typing import Dict
-
-from sqlalchemy import create_engine, text
-from sqlalchemy.engine import Engine
-from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError
-
-from shared_kernel.dataclasses.warehouse_configs import FabricSQLConfig
-from shared_kernel.exceptions.http_exceptions import BadRequest
-from shared_kernel.interfaces.warehouse_connection import DataWarehouseConnection
-from shared_kernel.data_warehouse_handlers.utils import extract_sql_server_error_message
-from shared_kernel.logger import Logger
-from shared_kernel.config import Config
-
-config = Config()
-logger = Logger(config.get("APP_NAME"))
-
-
-class FabricSQLWarehouseConnection(DataWarehouseConnection):
-
-    _engines: Dict[str, Engine] = {}
-    _lock = threading.Lock()
-
-    def __init__(
-        self,
-        source_config: FabricSQLConfig,
-        set_default_schema: bool = True,
-    ):
-        if source_config is None:
-            raise ValueError("Missing required parameter: source_config (FabricSQLConfig)")
-
-        self._fabric_config = source_config
-        self.org_schema = source_config.schema if source_config.schema else None
-        self.set_default_schema = set_default_schema
-
-        pagination = getattr(source_config, "pagination", None) or {}
-        self.page = pagination.get("page") if isinstance(pagination, dict) else None
-        self.limit = pagination.get("limit") if isinstance(pagination, dict) else None
-        self.search = pagination.get("search") if isinstance(pagination, dict) else None
-
-    def _engine_key(self, cfg: FabricSQLConfig) -> str:
-        pwd_hash = hashlib.sha256(cfg.password.encode()).hexdigest()
-        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}|{pwd_hash}"
-
-    def _get_fabric_connection_params(self, cfg: FabricSQLConfig) -> tuple:
-        """
-        Builds the token authentication and connection parameters for Microsoft Fabric SQL.
-        
-        Returns:
-            tuple: (connection_string, connect_args)
-        """
-        logger.info("Using Fabric SQL connector (pyodbc native) with access token authentication")
-        tenant_id = getattr(cfg, "tenant_id", None)
-        if not tenant_id:
-            raise ValueError(
-                "tenant_id is required in FabricSQLConfig for authenticating to Microsoft Fabric via Active Directory Service Principal."
-            )
-
-        from azure.identity import ClientSecretCredential
-        import struct
-
-        # Get OAuth2 token for Azure SQL / Fabric from Microsoft Entra ID
-        cred = ClientSecretCredential(
-            tenant_id=tenant_id,
-            client_id=cfg.username,
-            client_secret=cfg.password
-        )
-        token_obj = cred.get_token("https://database.windows.net/.default")
-        
-        # Encode token as UTF-16-LE and pack into structure expected by the SQL Server ODBC driver.
-        # The binary layout expects [4 bytes for length of token payload] followed by the [raw token bytes].
-        token_bytes = token_obj.token.encode("utf-16-le")
-        token_struct = struct.pack(f"<I{len(token_bytes)}s", len(token_bytes), token_bytes)
-
-        import urllib.parse
-        # Build ODBC params. We omit UID and Password fields entirely from the connection string.
-        # Authentication is handled solely by injecting the access token into the connection handle.
-        odbc_params = urllib.parse.quote_plus(
-            f"DRIVER={{ODBC Driver 18 for SQL Server}};"
-            f"SERVER={cfg.host},{int(cfg.port)};"
-            f"DATABASE={cfg.database};"
-            "Encrypt=yes;"
-            "TrustServerCertificate=no;"
-        )
-        # We use odbc_connect query param to prevent SQLAlchemy dialect from auto-injecting
-        # conflicting credentials/parameters (avoiding error FA005).
-        connection_string = f"mssql+pyodbc:///?odbc_connect={odbc_params}"
-        
-        # Inject the token using ODBC attribute SQL_COPT_SS_ACCESS_TOKEN (attribute ID 1256)
-        # before the connection is established.
-        connect_args = {"attrs_before": {1256: token_struct}}
-        return connection_string, connect_args
-
-    def _get_engine(self) -> Engine:
-        cfg = self._fabric_config
-        key = self._engine_key(cfg)
-
-        if key not in self._engines:
-            with self.__class__._lock: # thread-safe
-                if key not in self._engines: # recheck
-                    connection_string, connect_args = self._get_fabric_connection_params(cfg)
-
-                    self._engines[key] = create_engine(
-                        connection_string,
-                        connect_args=connect_args,
-                        pool_size=10,
-                        max_overflow=5,
-                        pool_timeout=30,
-                        pool_recycle=1800,
-                        pool_pre_ping=True,
-                    )
-                    logger.info(
-                        f"Created new Fabric SQL engine for {cfg.host}:{cfg.port}/{cfg.database}"
-                    )
-
-        return self._engines[key]
-
-    @contextmanager
-    def get_connection(self, retries: int = 3, delay: int = 5, query_timeout: int = 30):
-        """Context manager for database connections with retry mechanism"""
-        engine = self._get_engine()
-        last_exception = None
-
-        for attempt in range(retries):
-            conn = None
-            try:
-                conn = engine.connect()
-                yield conn
-                return
-
-            except (OperationalError, InterfaceError, ProgrammingError) as e:
-                last_exception = e
-                if conn is not None:
-                    conn.close()
-                    conn = None
-
-                error_message = extract_sql_server_error_message(str(e))
-                logger.warning(
-                    f"Fabric SQL connection failed (attempt {attempt + 1}/{retries}): {error_message}"
-                )
-
-                if attempt < retries - 1:
-                    time.sleep(delay)
-                    delay += 5
-                    continue
-
-                break
-
-            finally:
-                if conn is not None:
-                    conn.close()
-
-        raise BadRequest(
-            f"Failed to connect to Fabric SQL after {retries} attempts"
-        ) from last_exception
-
-    def close_connection(self):
-        """Dispose all pooled engines managed by this class."""
-        for engine in self._engines.values():
-            engine.dispose()
-        self._engines.clear()
-        logger.info("All Fabric SQL engine pools disposed.")
-
-    def is_valid_connection(self) -> bool:
-        """Pool health is managed by pool_pre_ping; no manual check needed."""
-        return True
-
-    def test_connection(self) -> bool:
-        """Verify the connection is reachable. Raises on failure."""
-        try:
-            with self.get_connection() as conn:
-                conn.execute(text("SELECT 1"))
-            return True
-        except OperationalError as e:
-            if "authentication failed" in str(e).lower():
-                raise ValueError("Invalid username or password")
-            raise ConnectionError(f"Database error: {str(e)}")
-        except InterfaceError:
-            raise ConnectionError("Cannot connect to database server")