fdc-shared-kernel 0.0.190__tar.gz → 0.0.192__tar.gz

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.192}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.190
+Version: 0.0.192
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.192}/fdc_shared_kernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.190
+Version: 0.0.192
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.192}/fdc_shared_kernel.egg-info/SOURCES.txt

@@ -36,6 +36,7 @@ shared_kernel/config/__init__.py
 shared_kernel/constants/constants.py
 shared_kernel/data_warehouse_handlers/__init__.py
 shared_kernel/data_warehouse_handlers/databricks_handler.py
+shared_kernel/data_warehouse_handlers/fabric_sql_handler.py
 shared_kernel/data_warehouse_handlers/factory.py
 shared_kernel/data_warehouse_handlers/jdbc_driver_registry.py
 shared_kernel/data_warehouse_handlers/jdbc_handler.py
@@ -48,6 +49,7 @@ shared_kernel/data_warehouse_handlers/snowflake_handler.py
 shared_kernel/data_warehouse_handlers/utils.py
 shared_kernel/data_warehouse_handlers/warehouse_handlers_map.py
 shared_kernel/data_warehouse_handlers/connections/databricks_warehouse_connection.py
+shared_kernel/data_warehouse_handlers/connections/fabric_sql_warehouse_connection.py
 shared_kernel/data_warehouse_handlers/connections/jdbc_warehouse_connection.py
 shared_kernel/data_warehouse_handlers/connections/mssql_warehouse_connection.py
 shared_kernel/data_warehouse_handlers/connections/mysql_warehouse_connection.py
@@ -56,6 +58,7 @@ shared_kernel/data_warehouse_handlers/connections/redshift_warehouse_connection.
 shared_kernel/data_warehouse_handlers/connections/snowflake_warehouse_connection.py
 shared_kernel/data_warehouse_handlers/custom_query_builders/databricks_query.py
 shared_kernel/data_warehouse_handlers/query_executors/databricks_query_executor.py
+shared_kernel/data_warehouse_handlers/query_executors/fabric_sql_query_executor.py
 shared_kernel/data_warehouse_handlers/query_executors/jdbc_query_executor.py
 shared_kernel/data_warehouse_handlers/query_executors/mssql_query_executor.py
 shared_kernel/data_warehouse_handlers/query_executors/mysql_query_executor.py

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.192}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fdc_shared_kernel"
-version = "0.0.190"
+version = "0.0.192"
 requires-python = ">=3.7"
 readme = "README_pypi.md"
 description = "Shared library for microservice"

fdc_shared_kernel-0.0.192/shared_kernel/data_warehouse_handlers/connections/fabric_sql_warehouse_connection.py

@@ -0,0 +1,218 @@
+import hashlib
+import time
+import threading
+
+from contextlib import contextmanager
+from typing import Dict
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError
+
+from shared_kernel.dataclasses.warehouse_configs import FabricSQLConfig
+from shared_kernel.exceptions.http_exceptions import BadRequest
+from shared_kernel.interfaces.warehouse_connection import DataWarehouseConnection
+from shared_kernel.data_warehouse_handlers.utils import extract_sql_server_error_message
+from shared_kernel.logger import Logger
+from shared_kernel.config import Config
+
+config = Config()
+logger = Logger(config.get("APP_NAME"))
+
+
+class FabricSQLWarehouseConnection(DataWarehouseConnection):
+
+    _engines: Dict[str, Engine] = {}
+    _tokens: Dict[str, dict] = {}
+    _lock = threading.Lock()
+
+    def __init__(
+        self,
+        source_config: FabricSQLConfig,
+        set_default_schema: bool = True,
+    ):
+        if source_config is None:
+            raise ValueError("Missing required parameter: source_config (FabricSQLConfig)")
+
+        self._fabric_config = source_config
+        self.org_schema = source_config.schema if source_config.schema else None
+        self.set_default_schema = set_default_schema
+
+        pagination = getattr(source_config, "pagination", None) or {}
+        self.page = pagination.get("page") if isinstance(pagination, dict) else None
+        self.limit = pagination.get("limit") if isinstance(pagination, dict) else None
+        self.search = pagination.get("search") if isinstance(pagination, dict) else None
+
+    def _engine_key(self, cfg: FabricSQLConfig) -> str:
+        h = hashlib.md5(f"{cfg.host}:{cfg.port}:{cfg.database}:{cfg.username}".encode()).hexdigest()
+        return h
+
+    def _get_fabric_connection_params(self, cfg: FabricSQLConfig) -> str:
+        """
+        Builds the base connection parameters for Microsoft Fabric SQL.
+        
+        Returns:
+            str: Connection string
+        """
+        import urllib.parse
+        # Build ODBC params. We omit UID and Password fields entirely from the connection string.
+        # Authentication is handled solely by injecting the access token into the connection handle.
+        odbc_params = urllib.parse.quote_plus(
+            f"DRIVER={{ODBC Driver 18 for SQL Server}};"
+            f"SERVER={cfg.host},{int(cfg.port)};"
+            f"DATABASE={cfg.database};"
+            "Encrypt=yes;"
+            "TrustServerCertificate=no;"
+        )
+        # We use odbc_connect query param to prevent SQLAlchemy dialect from auto-injecting
+        # conflicting credentials/parameters (avoiding error FA005).
+        connection_string = f"mssql+pyodbc:///?odbc_connect={odbc_params}"
+        return connection_string
+
+    def _get_engine(self) -> Engine:
+        cfg = self._fabric_config
+        key = self._engine_key(cfg)
+
+        if key not in self._engines:
+            with self.__class__._lock: # thread-safe
+                if key not in self._engines: # recheck
+                    from sqlalchemy import event
+                    
+                    connection_string = self._get_fabric_connection_params(cfg)
+
+                    engine = create_engine(
+                        connection_string,
+                        pool_size=10,
+                        max_overflow=5,
+                        pool_timeout=30,
+                        pool_recycle=1800,
+                        pool_pre_ping=True,
+                    )
+                    
+                    # Ensure a thread-safe token cache structure exists for this database key.
+                    # Each unique database configuration gets its own token cache, expiration tracker, and lock.
+                    if key not in self._tokens:
+                        self._tokens[key] = {
+                            "token": None,       # Stores the packed binary token structure
+                            "expires_at": 0,     # Unix timestamp indicating token expiration time
+                            "lock": threading.Lock() # Lock to synchronize concurrent token refreshes
+                        }
+                    
+                    # Register an event listener on the engine to intercept connection establishment.
+                    # Every time SQLAlchemy checks out or opens a new raw DBAPI connection from the pool,
+                    # this listener executes to inject a valid, fresh Entra ID access token.
+                    @event.listens_for(engine, "do_connect")
+                    def provide_token(dialect, conn_rec, cargs, cparams):
+                        now = time.time()
+                        token_info = self._tokens[key]
+                        
+                        # Check if the token is missing or near expiration (within 5 minutes / 300 seconds).
+                        # This buffer ensures we don't send a token that is about to expire.
+                        if token_info["token"] is None or now >= token_info["expires_at"] - 300:
+                            # Acquire the database-specific lock to serialize token generation.
+                            # If 10 threads try to connect concurrently when a token expires,
+                            # only the first thread executes the token fetch; others block here.
+                            with token_info["lock"]:
+                                # Double-checked locking pattern: once the lock is acquired,
+                                # re-verify if another thread updated the token while we were waiting.
+                                if token_info["token"] is None or now >= token_info["expires_at"] - 300:
+                                    tenant_id = getattr(cfg, "tenant_id", None)
+                                    if not tenant_id:
+                                        raise ValueError(
+                                            "tenant_id is required in FabricSQLConfig for authenticating to Microsoft Fabric via Active Directory Service Principal."
+                                        )
+                                    
+                                    from azure.identity import ClientSecretCredential
+                                    import struct
+                                    
+                                    logger.info("Fetching fresh Entra ID access token for Fabric SQL connection...")
+                                    # Instantiate the credential to retrieve a token for the Azure SQL Endpoint
+                                    cred = ClientSecretCredential(
+                                        tenant_id=tenant_id,
+                                        client_id=cfg.username,
+                                        client_secret=cfg.password
+                                    )
+                                    token_obj = cred.get_token("https://database.windows.net/.default")
+                                    
+                                    # Encode the OAuth token to UTF-16-LE bytes.
+                                    # Pack the payload into the binary structure required by ODBC Driver 18:
+                                    # [4 bytes representing the length of the token string] followed by [the token string bytes].
+                                    token_bytes = token_obj.token.encode("utf-16-le")
+                                    token_info["token"] = struct.pack(f"<I{len(token_bytes)}s", len(token_bytes), token_bytes)
+                                    token_info["expires_at"] = token_obj.expires_on
+                                    logger.info("Successfully fetched and cached fresh Entra ID access token.")
+                                    
+                        # Inject the packed binary token structure into pyodbc's attrs_before connection attribute list.
+                        # Attribute ID 1256 maps to SQL_COPT_SS_ACCESS_TOKEN.
+                        cparams["attrs_before"] = {1256: token_info["token"]}
+
+                    self._engines[key] = engine
+                    logger.info(
+                        f"Created new Fabric SQL engine for {cfg.host}:{cfg.port}/{cfg.database} with dynamic token provider"
+                    )
+
+        return self._engines[key]
+
+    @contextmanager
+    def get_connection(self, retries: int = 3, delay: int = 5, query_timeout: int = 30):
+        """Context manager for database connections with retry mechanism"""
+        engine = self._get_engine()
+        last_exception = None
+
+        for attempt in range(retries):
+            conn = None
+            try:
+                conn = engine.connect()
+                yield conn
+                return
+
+            except (OperationalError, InterfaceError, ProgrammingError) as e:
+                last_exception = e
+                if conn is not None:
+                    conn.close()
+                    conn = None
+
+                error_message = extract_sql_server_error_message(str(e))
+                logger.warning(
+                    f"Fabric SQL connection failed (attempt {attempt + 1}/{retries}): {error_message}"
+                )
+
+                if attempt < retries - 1:
+                    time.sleep(delay)
+                    delay += 5
+                    continue
+
+                break
+
+            finally:
+                if conn is not None:
+                    conn.close()
+
+        raise BadRequest(
+            f"Failed to connect to Fabric SQL after {retries} attempts"
+        ) from last_exception
+
+    def close_connection(self):
+        """Dispose all pooled engines managed by this class."""
+        for engine in self._engines.values():
+            engine.dispose()
+        self._engines.clear()
+        self._tokens.clear()
+        logger.info("All Fabric SQL engine pools disposed.")
+
+    def is_valid_connection(self) -> bool:
+        """Pool health is managed by pool_pre_ping; no manual check needed."""
+        return True
+
+    def test_connection(self) -> bool:
+        """Verify the connection is reachable. Raises on failure."""
+        try:
+            with self.get_connection() as conn:
+                conn.execute(text("SELECT 1"))
+            return True
+        except OperationalError as e:
+            if "authentication failed" in str(e).lower():
+                raise ValueError("Invalid username or password")
+            raise ConnectionError(f"Database error: {str(e)}")
+        except InterfaceError:
+            raise ConnectionError("Cannot connect to database server")

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.192}/shared_kernel/data_warehouse_handlers/connections/mssql_warehouse_connection.py

@@ -56,12 +56,15 @@ class MSSQLWarehouseConnection(DataWarehouseConnection):
             with self.__class__._lock: # thread-safe
                 if key not in self._engines: # recheck
                     password = quote(cfg.password)
+                    logger.info(f"Using MSSQL connector (pyodbc native) for standard SQL Server connection on {cfg.host}")
                     connection_string = get_constant("mssql", "CONNECTION_STRING").format(
                         cfg.username, password, cfg.host, int(cfg.port), cfg.database
                     )
+                    connect_args = {}
 
                     self._engines[key] = create_engine(
                         connection_string,
+                        connect_args=connect_args,
                         pool_size=10,
                         max_overflow=5,
                         pool_timeout=30,

fdc_shared_kernel-0.0.192/shared_kernel/data_warehouse_handlers/fabric_sql_handler.py

@@ -0,0 +1,75 @@
+from pypika import MSSQLQuery
+
+from shared_kernel.data_warehouse_handlers.connections.fabric_sql_warehouse_connection import (
+    FabricSQLWarehouseConnection,
+)
+from shared_kernel.data_warehouse_handlers.query_executors.fabric_sql_query_executor import (
+    FabricSQLQueryExecutor,
+)
+from shared_kernel.dataclasses.warehouse_configs import FabricSQLConfig
+from shared_kernel.interfaces.data_warehouse_handler import WarehouseHandler
+
+
+class FabricSQLWarehouseHandler(WarehouseHandler):
+
+    def __init__(self, payload: dict = None):
+        super().__init__(payload)
+        self.set_default_schema = self.payload.get("set_default_schema", True)
+
+    def get_config(self) -> FabricSQLConfig:
+        source_config: FabricSQLConfig = self.payload.get("source_config")
+        if source_config and isinstance(source_config, FabricSQLConfig):
+            return source_config
+
+        required = ("host", "user", "password", "database", "tenant_id")
+        missing = [f for f in required if not self.payload.get(f)]
+        if missing:
+            raise ValueError(f"Fabric SQL payload is missing required fields: {missing}.")
+
+        return FabricSQLConfig(
+            username=str(self.payload.get("user")).strip(),
+            password=str(self.payload.get("password")).strip(),
+            host=str(self.payload.get("host")).strip(),
+            database=str(self.payload.get("database")).strip(),
+            tenant_id=str(self.payload.get("tenant_id")).strip(),
+            port=str(self.payload.get("port", "1433")).strip(),
+            schema=(
+                str(self.payload.get("schema")).strip()
+                if self.payload.get("schema")
+                else None
+            ),
+            schema_list=self.payload.get("schema_list"),
+            pagination=self.pagination,
+        )
+
+    def _build_connection(self) -> FabricSQLWarehouseConnection:
+        return FabricSQLWarehouseConnection(
+            source_config=self.get_config(),
+            set_default_schema=self.set_default_schema,
+        )
+
+    def get_executor(self) -> FabricSQLQueryExecutor:
+        connection = self._build_connection()
+        return FabricSQLQueryExecutor(warehouse_connection=connection)
+
+    def get_query_object(self) -> MSSQLQuery:
+        return MSSQLQuery
+
+    def create_records_and_count_query(
+        self,
+        query: str,
+        limit: str,
+        page: str,
+    ) -> tuple[str, str]:
+        count_query = f"SELECT COUNT(*) FROM ({query}) AS _count"
+
+        if limit:
+            offset = (int(page) - 1) * int(limit) if page else 0
+            query = (
+                f"{query} "
+                f"ORDER BY (SELECT NULL) "
+                f"OFFSET {offset} ROWS "
+                f"FETCH NEXT {int(limit)} ROWS ONLY"
+            )
+
+        return count_query, query

fdc_shared_kernel-0.0.192/shared_kernel/data_warehouse_handlers/query_executors/fabric_sql_query_executor.py

@@ -0,0 +1,142 @@
+import time
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Dict, List, Optional, Tuple
+
+from sqlalchemy import text
+from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError
+
+from shared_kernel.config import Config
+from shared_kernel.data_warehouse_handlers.utils import extract_error_message
+from shared_kernel.interfaces.query_executor import DataWarehouseQueryExecutor
+from shared_kernel.interfaces.warehouse_connection import DataWarehouseConnection
+from shared_kernel.logger import Logger
+
+config = Config()
+logger = Logger(config.get("APP_NAME"))
+MAX_QUERY_CONCURRENCY = 5
+
+
+class FabricSQLQueryExecutor(DataWarehouseQueryExecutor):
+    """
+    Query executor for Microsoft Fabric SQL Server
+    """
+
+    def __init__(self, warehouse_connection: DataWarehouseConnection) -> None:
+        self.warehouse_connection = warehouse_connection
+
+    def execute_query(
+        self,
+        query: str,
+        params: Optional[tuple] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        Execute a single SQL query and return all rows as a list of dicts.
+        """
+        with self.warehouse_connection.get_connection() as conn:
+            try:
+                logger.info(f"Executing query: {query}")
+                start = time.perf_counter()
+
+                result = (
+                    conn.execute(text(query), params)
+                    if params
+                    else conn.execute(text(query))
+                )
+
+                elapsed = time.perf_counter() - start
+                logger.info(f"Query executed in {elapsed:.6f}s")
+
+                # Check if the query returns rows
+                if result.returns_rows:
+                    # Optimize fetch performance for Microsoft Fabric by batching
+                    result.cursor.arraysize = 5000
+                    columns = [col for col in result.keys()]
+                    return [dict(zip(columns, row)) for row in result.fetchall()]
+                else:
+                    # Return an empty list for DDL queries
+                    return []
+
+            except (OperationalError, ProgrammingError, InterfaceError) as e:
+                logger.error(f"Query failed: {query} — {e}", exc_info=True)
+                user_msg = extract_error_message(str(e))
+                raise Exception(f"Query execution failed: {user_msg}") from e
+            except Exception as e:
+                user_msg = extract_error_message(str(e))
+                raise Exception(f"Query execution failed: {user_msg}") from e
+
+    def execute_queries(
+        self,
+        queries: List[str],
+    ) -> List[List[Dict[str, Any]]]:
+        """
+        Execute multiple SQL queries in parallel and return results in input order.
+        """
+        if not queries:
+            return []
+
+        indexed: List[Tuple[int, str]] = list(enumerate(queries))
+        results: List[Optional[List[Dict[str, Any]]]] = [None] * len(queries)
+
+        with ThreadPoolExecutor(max_workers=MAX_QUERY_CONCURRENCY) as pool:
+            future_to_index = {
+                pool.submit(self.execute_query, query): idx
+                for idx, query in indexed
+            }
+
+            for future in as_completed(future_to_index):
+                idx = future_to_index[future]
+                try:
+                    results[idx] = future.result()
+                except Exception as e:
+                    # Cancel all pending futures
+                    for pending in future_to_index:
+                        if not pending.done():
+                            pending.cancel()
+                    raise Exception(
+                        f"Parallel query execution failed on query index {idx}: {e}"
+                    ) from e
+
+        return results
+
+    def execute_query_with_column_types(
+        self,
+        query: str,
+        params: Optional[tuple] = None,
+    ) -> Tuple[List[Dict[str, Any]], List[Tuple]]:
+        """
+        Execute a query and return both rows and raw DBAPI cursor description.
+        """
+        with self.warehouse_connection.get_connection() as conn:
+            try:
+                logger.info(f"Executing query with column types: {query}")
+                start = time.perf_counter()
+
+                result = (
+                    conn.execute(text(query), params)
+                    if params
+                    else conn.execute(text(query))
+                )
+
+                elapsed = time.perf_counter() - start
+                logger.info(f"Query executed in {elapsed:.6f}s")
+
+                # Check if the query returns rows
+                if result.returns_rows:
+                    # Optimize fetch performance for Microsoft Fabric by batching
+                    result.cursor.arraysize = 5000
+                    description = result.cursor.description
+                    columns = [col for col in result.keys()]
+                    rows = [dict(zip(columns, row)) for row in result.fetchall()]
+                    return rows, description
+                else:
+                    # Return empty rows and description for DDL queries
+                    return [], []
+
+            except (OperationalError, ProgrammingError, InterfaceError) as e:
+                logger.error(f"Query failed: {query} — {e}", exc_info=True)
+                user_msg = extract_error_message(str(e))
+                raise Exception(f"Query execution failed: {user_msg}") from e
+            except Exception as e:
+                user_msg = extract_error_message(str(e))
+                raise Exception(f"Query execution failed: {user_msg}") from e

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.192}/shared_kernel/data_warehouse_handlers/query_executors/mssql_query_executor.py

@@ -49,6 +49,7 @@ class MSSQLQueryExecutor(DataWarehouseQueryExecutor):
 
                 # Check if the query returns rows
                 if result.returns_rows:
+                    result.cursor.arraysize = 5000
                     columns = [col for col in result.keys()]
                     return [dict(zip(columns, row)) for row in result.fetchall()]
                 else:
@@ -128,6 +129,7 @@ class MSSQLQueryExecutor(DataWarehouseQueryExecutor):
 
                 # Check if the query returns rows
                 if result.returns_rows:
+                    result.cursor.arraysize = 5000
                     description = result.cursor.description
                     columns = [col for col in result.keys()]
                     rows = [dict(zip(columns, row)) for row in result.fetchall()]

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.192}/shared_kernel/data_warehouse_handlers/warehouse_handlers_map.py

@@ -7,6 +7,7 @@ from shared_kernel.data_warehouse_handlers.postgresql_handler import PostgreSQLW
 from shared_kernel.data_warehouse_handlers.redshift_handler import RedshiftWarehouseHandler
 from shared_kernel.data_warehouse_handlers.snowflake_handler import SnowflakeWarehouseHandler
 from shared_kernel.data_warehouse_handlers.jdbc_handler import JDBCWarehouseHandler
+from shared_kernel.data_warehouse_handlers.fabric_sql_handler import FabricSQLWarehouseHandler
 from shared_kernel.enums.connection_engine import ConnectionEngine
 
 
@@ -20,4 +21,5 @@ warehouse_handlers = {
     ConnectionEngine.AZURESQL.value: MSSQLWarehouseHandler,
     ConnectionEngine.DATABRICKS.value: DatabricksWarehouseHandler,
     ConnectionEngine.JDBC.value: JDBCWarehouseHandler,
+    ConnectionEngine.FABRIC_SQL.value: FabricSQLWarehouseHandler,
 }

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.192}/shared_kernel/dataclasses/warehouse_configs.py

@@ -46,6 +46,19 @@ class MSSQLConfig(BaseConfig):
     default_schema: str = "dbo"
 
 
+@dataclass
+class FabricSQLConfig(BaseConfig):
+    """Configuration for Microsoft Fabric SQL Server connections."""
+
+    username: str
+    password: str
+    host: str
+    database: str
+    tenant_id: str
+    port: str = "1433"
+    default_schema: str = "dbo"
+
+
 @dataclass
 class DatabricksConfig(BaseConfig):
     """Configuration for Databricks connections."""

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.192}/shared_kernel/enums/connection_engine.py

@@ -11,3 +11,4 @@ class ConnectionEngine(Enum):
     SNOWFLAKE = "snowflake"
     FACTWEAVERS_DW = "factweavers-dw"
     JDBC = "jdbc"
+    FABRIC_SQL = "fabric_sql"