fdc-shared-kernel 0.0.190__tar.gz → 0.0.191__tar.gz

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.191}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.190
+Version: 0.0.191
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.191}/fdc_shared_kernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.190
+Version: 0.0.191
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.191}/fdc_shared_kernel.egg-info/SOURCES.txt

@@ -36,6 +36,7 @@ shared_kernel/config/__init__.py
 shared_kernel/constants/constants.py
 shared_kernel/data_warehouse_handlers/__init__.py
 shared_kernel/data_warehouse_handlers/databricks_handler.py
+shared_kernel/data_warehouse_handlers/fabric_sql_handler.py
 shared_kernel/data_warehouse_handlers/factory.py
 shared_kernel/data_warehouse_handlers/jdbc_driver_registry.py
 shared_kernel/data_warehouse_handlers/jdbc_handler.py
@@ -48,6 +49,7 @@ shared_kernel/data_warehouse_handlers/snowflake_handler.py
 shared_kernel/data_warehouse_handlers/utils.py
 shared_kernel/data_warehouse_handlers/warehouse_handlers_map.py
 shared_kernel/data_warehouse_handlers/connections/databricks_warehouse_connection.py
+shared_kernel/data_warehouse_handlers/connections/fabric_sql_warehouse_connection.py
 shared_kernel/data_warehouse_handlers/connections/jdbc_warehouse_connection.py
 shared_kernel/data_warehouse_handlers/connections/mssql_warehouse_connection.py
 shared_kernel/data_warehouse_handlers/connections/mysql_warehouse_connection.py
@@ -56,6 +58,7 @@ shared_kernel/data_warehouse_handlers/connections/redshift_warehouse_connection.
 shared_kernel/data_warehouse_handlers/connections/snowflake_warehouse_connection.py
 shared_kernel/data_warehouse_handlers/custom_query_builders/databricks_query.py
 shared_kernel/data_warehouse_handlers/query_executors/databricks_query_executor.py
+shared_kernel/data_warehouse_handlers/query_executors/fabric_sql_query_executor.py
 shared_kernel/data_warehouse_handlers/query_executors/jdbc_query_executor.py
 shared_kernel/data_warehouse_handlers/query_executors/mssql_query_executor.py
 shared_kernel/data_warehouse_handlers/query_executors/mysql_query_executor.py

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.191}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fdc_shared_kernel"
-version = "0.0.190"
+version = "0.0.191"
 requires-python = ">=3.7"
 readme = "README_pypi.md"
 description = "Shared library for microservice"

fdc_shared_kernel-0.0.191/shared_kernel/data_warehouse_handlers/connections/fabric_sql_warehouse_connection.py

@@ -0,0 +1,183 @@
+import hashlib
+import time
+import threading
+
+from contextlib import contextmanager
+from typing import Dict
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError
+
+from shared_kernel.dataclasses.warehouse_configs import FabricSQLConfig
+from shared_kernel.exceptions.http_exceptions import BadRequest
+from shared_kernel.interfaces.warehouse_connection import DataWarehouseConnection
+from shared_kernel.data_warehouse_handlers.utils import extract_sql_server_error_message
+from shared_kernel.logger import Logger
+from shared_kernel.config import Config
+
+config = Config()
+logger = Logger(config.get("APP_NAME"))
+
+
+class FabricSQLWarehouseConnection(DataWarehouseConnection):
+
+    _engines: Dict[str, Engine] = {}
+    _lock = threading.Lock()
+
+    def __init__(
+        self,
+        source_config: FabricSQLConfig,
+        set_default_schema: bool = True,
+    ):
+        if source_config is None:
+            raise ValueError("Missing required parameter: source_config (FabricSQLConfig)")
+
+        self._fabric_config = source_config
+        self.org_schema = source_config.schema if source_config.schema else None
+        self.set_default_schema = set_default_schema
+
+        pagination = getattr(source_config, "pagination", None) or {}
+        self.page = pagination.get("page") if isinstance(pagination, dict) else None
+        self.limit = pagination.get("limit") if isinstance(pagination, dict) else None
+        self.search = pagination.get("search") if isinstance(pagination, dict) else None
+
+    def _engine_key(self, cfg: FabricSQLConfig) -> str:
+        pwd_hash = hashlib.sha256(cfg.password.encode()).hexdigest()
+        return f"{cfg.host}|{cfg.port}|{cfg.database}|{cfg.username}|{pwd_hash}"
+
+    def _get_fabric_connection_params(self, cfg: FabricSQLConfig) -> tuple:
+        """
+        Builds the token authentication and connection parameters for Microsoft Fabric SQL.
+        
+        Returns:
+            tuple: (connection_string, connect_args)
+        """
+        logger.info("Using Fabric SQL connector (pyodbc native) with access token authentication")
+        tenant_id = getattr(cfg, "tenant_id", None)
+        if not tenant_id:
+            raise ValueError(
+                "tenant_id is required in FabricSQLConfig for authenticating to Microsoft Fabric via Active Directory Service Principal."
+            )
+
+        from azure.identity import ClientSecretCredential
+        import struct
+
+        # Get OAuth2 token for Azure SQL / Fabric from Microsoft Entra ID
+        cred = ClientSecretCredential(
+            tenant_id=tenant_id,
+            client_id=cfg.username,
+            client_secret=cfg.password
+        )
+        token_obj = cred.get_token("https://database.windows.net/.default")
+        
+        # Encode token as UTF-16-LE and pack into structure expected by the SQL Server ODBC driver.
+        # The binary layout expects [4 bytes for length of token payload] followed by the [raw token bytes].
+        token_bytes = token_obj.token.encode("utf-16-le")
+        token_struct = struct.pack(f"<I{len(token_bytes)}s", len(token_bytes), token_bytes)
+
+        import urllib.parse
+        # Build ODBC params. We omit UID and Password fields entirely from the connection string.
+        # Authentication is handled solely by injecting the access token into the connection handle.
+        odbc_params = urllib.parse.quote_plus(
+            f"DRIVER={{ODBC Driver 18 for SQL Server}};"
+            f"SERVER={cfg.host},{int(cfg.port)};"
+            f"DATABASE={cfg.database};"
+            "Encrypt=yes;"
+            "TrustServerCertificate=no;"
+        )
+        # We use odbc_connect query param to prevent SQLAlchemy dialect from auto-injecting
+        # conflicting credentials/parameters (avoiding error FA005).
+        connection_string = f"mssql+pyodbc:///?odbc_connect={odbc_params}"
+        
+        # Inject the token using ODBC attribute SQL_COPT_SS_ACCESS_TOKEN (attribute ID 1256)
+        # before the connection is established.
+        connect_args = {"attrs_before": {1256: token_struct}}
+        return connection_string, connect_args
+
+    def _get_engine(self) -> Engine:
+        cfg = self._fabric_config
+        key = self._engine_key(cfg)
+
+        if key not in self._engines:
+            with self.__class__._lock: # thread-safe
+                if key not in self._engines: # recheck
+                    connection_string, connect_args = self._get_fabric_connection_params(cfg)
+
+                    self._engines[key] = create_engine(
+                        connection_string,
+                        connect_args=connect_args,
+                        pool_size=10,
+                        max_overflow=5,
+                        pool_timeout=30,
+                        pool_recycle=1800,
+                        pool_pre_ping=True,
+                    )
+                    logger.info(
+                        f"Created new Fabric SQL engine for {cfg.host}:{cfg.port}/{cfg.database}"
+                    )
+
+        return self._engines[key]
+
+    @contextmanager
+    def get_connection(self, retries: int = 3, delay: int = 5, query_timeout: int = 30):
+        """Context manager for database connections with retry mechanism"""
+        engine = self._get_engine()
+        last_exception = None
+
+        for attempt in range(retries):
+            conn = None
+            try:
+                conn = engine.connect()
+                yield conn
+                return
+
+            except (OperationalError, InterfaceError, ProgrammingError) as e:
+                last_exception = e
+                if conn is not None:
+                    conn.close()
+                    conn = None
+
+                error_message = extract_sql_server_error_message(str(e))
+                logger.warning(
+                    f"Fabric SQL connection failed (attempt {attempt + 1}/{retries}): {error_message}"
+                )
+
+                if attempt < retries - 1:
+                    time.sleep(delay)
+                    delay += 5
+                    continue
+
+                break
+
+            finally:
+                if conn is not None:
+                    conn.close()
+
+        raise BadRequest(
+            f"Failed to connect to Fabric SQL after {retries} attempts"
+        ) from last_exception
+
+    def close_connection(self):
+        """Dispose all pooled engines managed by this class."""
+        for engine in self._engines.values():
+            engine.dispose()
+        self._engines.clear()
+        logger.info("All Fabric SQL engine pools disposed.")
+
+    def is_valid_connection(self) -> bool:
+        """Pool health is managed by pool_pre_ping; no manual check needed."""
+        return True
+
+    def test_connection(self) -> bool:
+        """Verify the connection is reachable. Raises on failure."""
+        try:
+            with self.get_connection() as conn:
+                conn.execute(text("SELECT 1"))
+            return True
+        except OperationalError as e:
+            if "authentication failed" in str(e).lower():
+                raise ValueError("Invalid username or password")
+            raise ConnectionError(f"Database error: {str(e)}")
+        except InterfaceError:
+            raise ConnectionError("Cannot connect to database server")

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.191}/shared_kernel/data_warehouse_handlers/connections/mssql_warehouse_connection.py

@@ -56,12 +56,15 @@ class MSSQLWarehouseConnection(DataWarehouseConnection):
             with self.__class__._lock: # thread-safe
                 if key not in self._engines: # recheck
                     password = quote(cfg.password)
+                    logger.info(f"Using MSSQL connector (pyodbc native) for standard SQL Server connection on {cfg.host}")
                     connection_string = get_constant("mssql", "CONNECTION_STRING").format(
                         cfg.username, password, cfg.host, int(cfg.port), cfg.database
                     )
+                    connect_args = {}
 
                     self._engines[key] = create_engine(
                         connection_string,
+                        connect_args=connect_args,
                         pool_size=10,
                         max_overflow=5,
                         pool_timeout=30,

fdc_shared_kernel-0.0.191/shared_kernel/data_warehouse_handlers/fabric_sql_handler.py

@@ -0,0 +1,78 @@
+from pypika import MSSQLQuery
+
+from shared_kernel.data_warehouse_handlers.connections.fabric_sql_warehouse_connection import (
+    FabricSQLWarehouseConnection,
+)
+from shared_kernel.data_warehouse_handlers.query_executors.fabric_sql_query_executor import (
+    FabricSQLQueryExecutor,
+)
+from shared_kernel.dataclasses.warehouse_configs import FabricSQLConfig
+from shared_kernel.interfaces.data_warehouse_handler import WarehouseHandler
+
+
+class FabricSQLWarehouseHandler(WarehouseHandler):
+
+    def __init__(self, payload: dict = None):
+        super().__init__(payload)
+        self.set_default_schema = self.payload.get("set_default_schema", True)
+
+    def get_config(self) -> FabricSQLConfig:
+        source_config: FabricSQLConfig = self.payload.get("source_config")
+        if source_config and isinstance(source_config, FabricSQLConfig):
+            return source_config
+
+        required = ("host", "user", "password", "database", "tenant_id")
+        missing = [f for f in required if not self.payload.get(f)]
+        if missing:
+            raise ValueError(f"Fabric SQL payload is missing required fields: {missing}.")
+
+        return FabricSQLConfig(
+            username=str(self.payload.get("user")).strip(),
+            password=str(self.payload.get("password")).strip(),
+            host=str(self.payload.get("host")).strip(),
+            database=str(self.payload.get("database")).strip(),
+            tenant_id=str(self.payload.get("tenant_id")).strip(),
+            port=str(self.payload.get("port", "1433")).strip(),
+            schema=(
+                str(self.payload.get("schema")).strip()
+                if self.payload.get("schema")
+                else None
+            ),
+            schema_list=self.payload.get("schema_list"),
+            pagination=self.pagination,
+        )
+
+    def _build_connection(self) -> FabricSQLWarehouseConnection:
+        return FabricSQLWarehouseConnection(
+            source_config=self.get_config(),
+            set_default_schema=self.set_default_schema,
+        )
+
+    def get_executor(self) -> FabricSQLQueryExecutor:
+        connection = self._build_connection()
+        return FabricSQLQueryExecutor(warehouse_connection=connection)
+
+    def get_constants_section(self) -> str:
+        return "mssql"
+
+    def get_query_object(self) -> MSSQLQuery:
+        return MSSQLQuery
+
+    def create_records_and_count_query(
+        self,
+        query: str,
+        limit: str,
+        page: str,
+    ) -> tuple[str, str]:
+        count_query = f"SELECT COUNT(*) FROM ({query}) AS _count"
+
+        if limit:
+            offset = (int(page) - 1) * int(limit) if page else 0
+            query = (
+                f"{query} "
+                f"ORDER BY (SELECT NULL) "
+                f"OFFSET {offset} ROWS "
+                f"FETCH NEXT {int(limit)} ROWS ONLY"
+            )
+
+        return count_query, query

fdc_shared_kernel-0.0.191/shared_kernel/data_warehouse_handlers/query_executors/fabric_sql_query_executor.py

@@ -0,0 +1,142 @@
+import time
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Dict, List, Optional, Tuple
+
+from sqlalchemy import text
+from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError
+
+from shared_kernel.config import Config
+from shared_kernel.data_warehouse_handlers.utils import extract_error_message
+from shared_kernel.interfaces.query_executor import DataWarehouseQueryExecutor
+from shared_kernel.interfaces.warehouse_connection import DataWarehouseConnection
+from shared_kernel.logger import Logger
+
+config = Config()
+logger = Logger(config.get("APP_NAME"))
+MAX_QUERY_CONCURRENCY = 5
+
+
+class FabricSQLQueryExecutor(DataWarehouseQueryExecutor):
+    """
+    Query executor for Microsoft Fabric SQL Server
+    """
+
+    def __init__(self, warehouse_connection: DataWarehouseConnection) -> None:
+        self.warehouse_connection = warehouse_connection
+
+    def execute_query(
+        self,
+        query: str,
+        params: Optional[tuple] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        Execute a single SQL query and return all rows as a list of dicts.
+        """
+        with self.warehouse_connection.get_connection() as conn:
+            try:
+                logger.info(f"Executing query: {query}")
+                start = time.perf_counter()
+
+                result = (
+                    conn.execute(text(query), params)
+                    if params
+                    else conn.execute(text(query))
+                )
+
+                elapsed = time.perf_counter() - start
+                logger.info(f"Query executed in {elapsed:.6f}s")
+
+                # Check if the query returns rows
+                if result.returns_rows:
+                    # Optimize fetch performance for Microsoft Fabric by batching
+                    result.cursor.arraysize = 5000
+                    columns = [col for col in result.keys()]
+                    return [dict(zip(columns, row)) for row in result.fetchall()]
+                else:
+                    # Return an empty list for DDL queries
+                    return []
+
+            except (OperationalError, ProgrammingError, InterfaceError) as e:
+                logger.error(f"Query failed: {query} — {e}", exc_info=True)
+                user_msg = extract_error_message(str(e))
+                raise Exception(f"Query execution failed: {user_msg}") from e
+            except Exception as e:
+                user_msg = extract_error_message(str(e))
+                raise Exception(f"Query execution failed: {user_msg}") from e
+
+    def execute_queries(
+        self,
+        queries: List[str],
+    ) -> List[List[Dict[str, Any]]]:
+        """
+        Execute multiple SQL queries in parallel and return results in input order.
+        """
+        if not queries:
+            return []
+
+        indexed: List[Tuple[int, str]] = list(enumerate(queries))
+        results: List[Optional[List[Dict[str, Any]]]] = [None] * len(queries)
+
+        with ThreadPoolExecutor(max_workers=MAX_QUERY_CONCURRENCY) as pool:
+            future_to_index = {
+                pool.submit(self.execute_query, query): idx
+                for idx, query in indexed
+            }
+
+            for future in as_completed(future_to_index):
+                idx = future_to_index[future]
+                try:
+                    results[idx] = future.result()
+                except Exception as e:
+                    # Cancel all pending futures
+                    for pending in future_to_index:
+                        if not pending.done():
+                            pending.cancel()
+                    raise Exception(
+                        f"Parallel query execution failed on query index {idx}: {e}"
+                    ) from e
+
+        return results
+
+    def execute_query_with_column_types(
+        self,
+        query: str,
+        params: Optional[tuple] = None,
+    ) -> Tuple[List[Dict[str, Any]], List[Tuple]]:
+        """
+        Execute a query and return both rows and raw DBAPI cursor description.
+        """
+        with self.warehouse_connection.get_connection() as conn:
+            try:
+                logger.info(f"Executing query with column types: {query}")
+                start = time.perf_counter()
+
+                result = (
+                    conn.execute(text(query), params)
+                    if params
+                    else conn.execute(text(query))
+                )
+
+                elapsed = time.perf_counter() - start
+                logger.info(f"Query executed in {elapsed:.6f}s")
+
+                # Check if the query returns rows
+                if result.returns_rows:
+                    # Optimize fetch performance for Microsoft Fabric by batching
+                    result.cursor.arraysize = 5000
+                    description = result.cursor.description
+                    columns = [col for col in result.keys()]
+                    rows = [dict(zip(columns, row)) for row in result.fetchall()]
+                    return rows, description
+                else:
+                    # Return empty rows and description for DDL queries
+                    return [], []
+
+            except (OperationalError, ProgrammingError, InterfaceError) as e:
+                logger.error(f"Query failed: {query} — {e}", exc_info=True)
+                user_msg = extract_error_message(str(e))
+                raise Exception(f"Query execution failed: {user_msg}") from e
+            except Exception as e:
+                user_msg = extract_error_message(str(e))
+                raise Exception(f"Query execution failed: {user_msg}") from e

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.191}/shared_kernel/data_warehouse_handlers/query_executors/mssql_query_executor.py

@@ -49,6 +49,7 @@ class MSSQLQueryExecutor(DataWarehouseQueryExecutor):
 
                 # Check if the query returns rows
                 if result.returns_rows:
+                    result.cursor.arraysize = 5000
                     columns = [col for col in result.keys()]
                     return [dict(zip(columns, row)) for row in result.fetchall()]
                 else:
@@ -128,6 +129,7 @@ class MSSQLQueryExecutor(DataWarehouseQueryExecutor):
 
                 # Check if the query returns rows
                 if result.returns_rows:
+                    result.cursor.arraysize = 5000
                     description = result.cursor.description
                     columns = [col for col in result.keys()]
                     rows = [dict(zip(columns, row)) for row in result.fetchall()]

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.191}/shared_kernel/data_warehouse_handlers/warehouse_handlers_map.py

@@ -7,6 +7,7 @@ from shared_kernel.data_warehouse_handlers.postgresql_handler import PostgreSQLW
 from shared_kernel.data_warehouse_handlers.redshift_handler import RedshiftWarehouseHandler
 from shared_kernel.data_warehouse_handlers.snowflake_handler import SnowflakeWarehouseHandler
 from shared_kernel.data_warehouse_handlers.jdbc_handler import JDBCWarehouseHandler
+from shared_kernel.data_warehouse_handlers.fabric_sql_handler import FabricSQLWarehouseHandler
 from shared_kernel.enums.connection_engine import ConnectionEngine
 
 
@@ -20,4 +21,5 @@ warehouse_handlers = {
     ConnectionEngine.AZURESQL.value: MSSQLWarehouseHandler,
     ConnectionEngine.DATABRICKS.value: DatabricksWarehouseHandler,
     ConnectionEngine.JDBC.value: JDBCWarehouseHandler,
+    ConnectionEngine.FABRIC_SQL.value: FabricSQLWarehouseHandler,
 }

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.191}/shared_kernel/dataclasses/warehouse_configs.py

@@ -46,6 +46,19 @@ class MSSQLConfig(BaseConfig):
     default_schema: str = "dbo"
 
 
+@dataclass
+class FabricSQLConfig(BaseConfig):
+    """Configuration for Microsoft Fabric SQL Server connections."""
+
+    username: str
+    password: str
+    host: str
+    database: str
+    tenant_id: str
+    port: str = "1433"
+    default_schema: str = "dbo"
+
+
 @dataclass
 class DatabricksConfig(BaseConfig):
     """Configuration for Databricks connections."""

{fdc_shared_kernel-0.0.190 → fdc_shared_kernel-0.0.191}/shared_kernel/enums/connection_engine.py

@@ -11,3 +11,4 @@ class ConnectionEngine(Enum):
     SNOWFLAKE = "snowflake"
     FACTWEAVERS_DW = "factweavers-dw"
     JDBC = "jdbc"
+    FABRIC_SQL = "fabric_sql"