binaryrain-fabric-local-pyspark 0.1.2__tar.gz

binaryrain_fabric_local_pyspark-0.1.2/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.3
+Name: binaryrain-fabric-local-pyspark
+Version: 0.1.2
+Summary: This tool can be used for local fabric development by providing a local caching layer to reduce capacity usage.
+Keywords: binary rain,common,help,functions,data processing,fabric,local,pyspark
+Author: Binary Rain, Marcel T.O
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: azure-identity>=1.26.0
+Requires-Dist: azure-storage-file-datalake>=12.26.0
+Requires-Dist: deltalake>=1.6.6
+Requires-Dist: pandas>=3.0.6
+Requires-Dist: pyarrow>=26.0.0
+Requires-Dist: pycryptodome>=3.24.1
+Requires-Dist: pyodbc>=5.3.0
+Requires-Dist: pyspark>=4.2.0
+Requires-Dist: python-dotenv>=1.2.4
+Requires-Dist: sqlalchemy>=2.1.4
+Requires-Python: >=3.12
+Project-URL: Homepage, https://binaryrain-net.github.io/Binary-Rain-Helper-Toolkit/
+Description-Content-Type: text/markdown
+

binaryrain_fabric_local_pyspark-0.1.2/pyproject.toml

@@ -0,0 +1,49 @@
+[project]
+name = "binaryrain-fabric-local-pyspark"
+version = "0.1.2"
+description = "This tool can be used for local fabric development by providing a local caching layer to reduce capacity usage."
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "azure-identity>=1.26.0",
+    "azure-storage-file-datalake>=12.26.0",
+    "deltalake>=1.6.6",
+    "pandas>=3.0.6",
+    "pyarrow>=26.0.0",
+    "pycryptodome>=3.24.1",
+    "pyodbc>=5.3.0",
+    "pyspark>=4.2.0",
+    "python-dotenv>=1.2.4",
+    "sqlalchemy>=2.1.4",
+]
+keywords = [
+    "binary rain",
+    "common",
+    "help",
+    "functions",
+    "data processing",
+    "fabric",
+    "local",
+    "pyspark",
+]
+classifiers = [
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+]
+
+[[project.authors]]
+name = "Binary Rain"
+
+[[project.authors]]
+name = "Marcel T.O"
+
+[project.urls]
+Homepage = "https://binaryrain-net.github.io/Binary-Rain-Helper-Toolkit/"
+
+[tool.ruff]
+extend = "../../ruff.toml"
+
+[build-system]
+requires = ["uv_build>=0.11.24,<0.12.0"]
+build-backend = "uv_build"

binaryrain_fabric_local_pyspark-0.1.2/pyproject.toml.orig

@@ -0,0 +1,47 @@
+[project]
+name = "binaryrain-fabric-local-pyspark"
+version = "0.1.2"
+description = "This tool can be used for local fabric development by providing a local caching layer to reduce capacity usage."
+readme = "README.md"
+authors = [{ name = "Binary Rain" }, { name = "Marcel T.O" }]
+requires-python = ">=3.12"
+dependencies = [
+    "azure-identity>=1.26.0",
+    "azure-storage-file-datalake>=12.26.0",
+    "deltalake>=1.6.6",
+    "pandas>=3.0.6",
+    "pyarrow>=26.0.0",
+    "pycryptodome>=3.24.1",
+    "pyodbc>=5.3.0",
+    "pyspark>=4.2.0",
+    "python-dotenv>=1.2.4",
+    "sqlalchemy>=2.1.4",
+]
+
+keywords = [
+    "binary rain",
+    "common",
+    "help",
+    "functions",
+    "data processing",
+    "fabric",
+    "local",
+    "pyspark",
+]
+
+classifiers = [
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+]
+
+[project.urls]
+Homepage = "https://binaryrain-net.github.io/Binary-Rain-Helper-Toolkit/"
+
+[tool.ruff]
+# Extend the settings from the ruff.toml file.
+extend = "../../ruff.toml"
+
+[build-system]
+requires = ["uv_build>=0.11.24,<0.12.0"]
+build-backend = "uv_build"

binaryrain_fabric_local_pyspark-0.1.2/src/binaryrain_fabric_local_pyspark/__init__.py

@@ -0,0 +1 @@
+"""Local Spark and Fabric lakehouse caching helpers."""

binaryrain_fabric_local_pyspark-0.1.2/src/binaryrain_fabric_local_pyspark/auth.py

@@ -0,0 +1,18 @@
+"""Create Azure credentials and access tokens for OneLake requests."""
+
+from functools import cache
+
+from azure.identity import DefaultAzureCredential
+
+from binaryrain_helper_local_pyspark.config import STORAGE_SCOPE
+
+
+@cache
+def _credential() -> DefaultAzureCredential:
+    """Return the process-wide Azure credential used for OneLake access."""
+    return DefaultAzureCredential()
+
+
+def _storage_token() -> str:
+    """Return an Azure Storage bearer token for OneLake requests."""
+    return _credential().get_token(STORAGE_SCOPE).token

binaryrain_fabric_local_pyspark-0.1.2/src/binaryrain_fabric_local_pyspark/config.py

@@ -0,0 +1,24 @@
+"""Configuration constants and environment variables used by the package.
+
+``FABRIC_CACHE_DIR`` overrides the cache root. When unset, cached data goes under
+``.cache/fabric`` relative to the package source tree. Set ``FABRIC_CACHE_REFRESH=1``
+to refresh cached data. ``FABRIC_WORKSPACE_ID`` and ``FABRIC_LAKEHOUSE_ID`` identify
+the default lakehouse for relative paths. ``FABRIC_SERVER`` and ``FABRIC_DATABASE``
+configure the SQL endpoint, and ``FABRIC_SPARK_DEBUG=1`` enables verbose Spark output.
+"""
+
+import os
+from pathlib import Path
+
+CACHE_DIR = Path(os.getenv("FABRIC_CACHE_DIR", Path(__file__).parents[1] / ".cache" / "fabric"))
+
+LAKEHOUSE_AREAS = ("Files", "Tables")
+MIN_ABFSS_SEGMENTS = 3  # <lakehouse>/<area>/<relative>
+MIN_RELATIVE_SEGMENTS = 2  # <area>/<relative>
+
+ONELAKE_HOST = "onelake.dfs.fabric.microsoft.com"
+STORAGE_SCOPE = "https://storage.azure.com/.default"
+FABRIC_TOKEN_SCOPE = "https://database.windows.net/.default"
+# SQL_COPT_SS_ACCESS_TOKEN connection attribute for passing an AAD access token
+# to the ODBC Driver for SQL Server.
+SQL_COPT_SS_ACCESS_TOKEN = 1256

binaryrain_fabric_local_pyspark-0.1.2/src/binaryrain_fabric_local_pyspark/fabric.py

@@ -0,0 +1,222 @@
+"""Parse Fabric lakehouse paths and cache their data on the local filesystem."""
+
+import logging
+import os
+import shutil
+from pathlib import Path, PurePosixPath
+from urllib.parse import urlsplit
+
+import pyarrow.dataset as ds
+from azure.core.exceptions import ResourceNotFoundError
+from azure.storage.filedatalake import DataLakeServiceClient
+from deltalake import DeltaTable
+
+from binaryrain_helper_local_pyspark.auth import _credential, _storage_token
+from binaryrain_helper_local_pyspark.config import (
+    LAKEHOUSE_AREAS,
+    MIN_ABFSS_SEGMENTS,
+    MIN_RELATIVE_SEGMENTS,
+)
+from binaryrain_helper_local_pyspark.schemas import FabricPath
+
+logger = logging.getLogger(__name__)
+
+
+def parse_fabric_path(path: str) -> FabricPath | None:
+    """
+    Parse an abfss URI or a default lakehouse relative path into its parts.
+
+    ``abfss://<workspace>@<host>/<lakehouse>/<Files|Tables>/<relative>`` is split directly.
+    ``Files/<relative>`` and ``Tables/<relative>`` are resolved against the default lakehouse
+    from ``FABRIC_WORKSPACE_ID`` and ``FABRIC_LAKEHOUSE_ID``.
+
+    :param str path:
+        An abfss URI, a default lakehouse-relative path, or a local path.
+
+    :returns FabricPath | None:
+        Parsed Fabric path, or None when the input is not a recognized Fabric path.
+    exception : ValueError
+        If an abfss URI is malformed or the default lakehouse IDs are missing for a relative path.
+    """
+    if path.lower().startswith("abfss://"):
+        parts = urlsplit(path)
+        workspace, _, host = parts.netloc.partition("@")
+        segments = PurePosixPath(parts.path).parts[1:]
+        if (
+            not workspace
+            or not host
+            or len(segments) < MIN_ABFSS_SEGMENTS
+            or segments[1] not in LAKEHOUSE_AREAS
+        ):
+            raise ValueError(
+                f"Expected abfss://<workspace>@<host>/<lakehouse>/<Files|Tables>/..., got {path}"
+            )
+        return FabricPath(workspace, segments[0], segments[1], "/".join(segments[2:]), host)
+
+    segments = PurePosixPath(path.replace("\\", "/").lstrip("/")).parts
+    if len(segments) < MIN_RELATIVE_SEGMENTS or segments[0] not in LAKEHOUSE_AREAS:
+        return None
+
+    workspace = os.getenv("FABRIC_WORKSPACE_ID")
+    lakehouse = os.getenv("FABRIC_LAKEHOUSE_ID")
+    if not workspace or not lakehouse:
+        raise ValueError(
+            f"{path} uses the default lakehouse, so FABRIC_WORKSPACE_ID and "
+            "FABRIC_LAKEHOUSE_ID must be set"
+        )
+    return FabricPath(workspace, lakehouse, segments[0], "/".join(segments[1:]))
+
+
+def resolve_local_path(path: str, refresh: bool | None = None) -> str:
+    """
+    Return a local path for ``path``, downloading Fabric data into the cache when needed.
+
+    :param str path:
+        abfss URI, default lakehouse relative path, or local path.
+    :param bool | None refresh:
+        Refresh the cache when True or use ``FABRIC_CACHE_REFRESH`` when None. False uses the
+        existing cache when available.
+
+    :returns str:
+        Local cached path for a Fabric path, or the original path for any other input.
+    exception : ValueError
+        If a default lakehouse-relative path is used but its workspace or lakehouse ID is unset.
+    """
+    fabric_path = parse_fabric_path(path)
+    if fabric_path is None:
+        return path
+    if refresh is None:
+        refresh = os.getenv("FABRIC_CACHE_REFRESH") == "1"
+    if fabric_path.area == "Tables":
+        return str(cache_table(fabric_path, refresh))
+    return str(cache_file(fabric_path, refresh))
+
+
+def cache_table(table: FabricPath | str, refresh: bool = False) -> Path:
+    """
+    Cache a lakehouse Delta table locally as parquet and return its cache directory.
+
+    A cache hit is returned without checking the remote table unless ``refresh`` is True.
+    Refreshing replaces the cached directory with data from the current table version.
+
+    :param FabricPath | str table:
+        A ``Tables`` path, such as ``Tables/dbo/kunden``, or a full abfss URI.
+    :param bool refresh:
+        Download again even when a cached copy exists. Default is False.
+
+    :returns Path:
+        Path to the directory containing the table's cached parquet files.
+    exception : ValueError
+        If the path is not a lakehouse ``Tables`` path.
+    exception : deltalake.exceptions.TableNotFoundError
+        If the remote Delta table does not exist.
+    """
+    fabric_path = _require_area(table, "Tables")
+    target = fabric_path.local_path
+    if target.exists() and not refresh:
+        logger.info("Cache hit, reading local copy: %s", target)
+        return target
+
+    logger.info("Cache miss, downloading from Fabric: %s", fabric_path.uri)
+    options = {"bearer_token": _storage_token(), "use_fabric_endpoint": "true"}
+    dataset = DeltaTable(fabric_path.uri, storage_options=options).to_pyarrow_dataset()
+
+    tmp_path = target.with_name(f"{target.name}.tmp")
+    shutil.rmtree(tmp_path, ignore_errors=True)
+    ds.write_dataset(dataset, tmp_path, format="parquet")
+    _commit(tmp_path, target)
+    return target
+
+
+def cache_file(file: FabricPath | str, refresh: bool = False) -> Path:
+    """
+    Cache a file or folder below the lakehouse ``Files`` area and return its local path.
+
+    A cache hit is returned without checking the remote item unless ``refresh`` is True.
+
+    :param FabricPath | str file:
+        A ``Files`` path, such as ``Files/onPrem/mssql/Kunden.parquet``, or a full abfss URI.
+    :param bool refresh:
+        Download again even when a cached copy exists. Default is False.
+
+    :returns Path:
+        Path to the cached file or folder.
+    exception : ValueError
+        If the path is not a lakehouse ``Files`` path.
+    exception : FileNotFoundError
+        If the remote file or folder does not exist.
+    """
+    fabric_path = _require_area(file, "Files")
+    target = fabric_path.local_path
+    if target.exists() and not refresh:
+        logger.info("Cache hit, reading local copy: %s", target)
+        return target
+
+    service = DataLakeServiceClient(f"https://{fabric_path.host}", credential=_credential())
+    file_system = service.get_file_system_client(fabric_path.workspace)
+    remote_root = fabric_path.lakehouse_path
+    logger.info("Cache miss, downloading from Fabric: %s", fabric_path.uri)
+
+    try:
+        properties = file_system.get_file_client(remote_root).get_file_properties()
+    except ResourceNotFoundError as exc:
+        raise FileNotFoundError(f"{exc.error_code}: {fabric_path.uri} not found") from exc
+
+    # Spark writes "x.parquet" as a folder, so both layouts must be supported.
+    is_folder = properties.metadata.get("hdi_isfolder") == "true"
+    if is_folder:
+        remote_files = [
+            item.name for item in file_system.get_paths(remote_root) if not item.is_directory
+        ]
+    else:
+        remote_files = [remote_root]
+
+    tmp_path = target.with_name(f"{target.name}.tmp")
+    shutil.rmtree(tmp_path, ignore_errors=True)
+    for remote_file in remote_files:
+        local_file = tmp_path
+        if is_folder:
+            local_file = tmp_path / PurePosixPath(remote_file).relative_to(remote_root)
+        local_file.parent.mkdir(parents=True, exist_ok=True)
+        with local_file.open("wb") as handle:
+            file_system.get_file_client(remote_file).download_file().readinto(handle)
+
+    _commit(tmp_path, target)
+    return target
+
+
+def _require_area(path: FabricPath | str, area: str) -> FabricPath:
+    """
+    Parse ``path`` and check that it points into the expected lakehouse area.
+
+    :param FabricPath | str path:
+        Parsed path or a string accepted by ``parse_fabric_path``.
+    :param str area:
+        Required lakehouse area, either ``Files`` or ``Tables``.
+
+    :returns FabricPath:
+        Parsed lakehouse path.
+    exception : ValueError
+        If the path is not a Fabric path or points into another area.
+    """
+    fabric_path = path if isinstance(path, FabricPath) else parse_fabric_path(path)
+    if fabric_path is None or fabric_path.area != area:
+        raise ValueError(f"Expected a lakehouse {area} path, got {path}")
+    return fabric_path
+
+
+def _commit(tmp_path: Path, target: Path) -> None:
+    """
+    Replace the target with a completed download from a sibling temporary path.
+
+    :param Path tmp_path:
+        Temporary location holding the completed download.
+    :param Path target:
+        Final cache location.
+    """
+    if target.is_dir():
+        shutil.rmtree(target)
+    elif target.exists():
+        target.unlink()
+    target.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path.rename(target)

binaryrain_fabric_local_pyspark-0.1.2/src/binaryrain_fabric_local_pyspark/schemas.py

@@ -0,0 +1,44 @@
+"""Data structures for representing Fabric lakehouse paths."""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from binaryrain_helper_local_pyspark.config import CACHE_DIR, ONELAKE_HOST
+
+
+@dataclass(frozen=True, slots=True)
+class FabricPath:
+    """Represent a location inside a Fabric lakehouse.
+
+    :param str workspace:
+        Workspace ID or name, which identifies the abfss file-system container.
+    :param str lakehouse:
+        Lakehouse ID or ``<name>.Lakehouse``.
+    :param str area:
+        Either ``Files`` or ``Tables``.
+    :param str relative:
+        Path below the area, for example ``onPrem/mssql/Kunden.parquet`` or ``dbo/kunden``.
+    :param str host:
+        OneLake host name. Default is the global OneLake endpoint.
+    """
+
+    workspace: str
+    lakehouse: str
+    area: str
+    relative: str
+    host: str = ONELAKE_HOST
+
+    @property
+    def lakehouse_path(self) -> str:
+        """Return the path below the workspace container, beginning with the lakehouse."""
+        return f"{self.lakehouse}/{self.area}/{self.relative}"
+
+    @property
+    def uri(self) -> str:
+        """Return the full abfss URI for this lakehouse location."""
+        return f"abfss://{self.workspace}@{self.host}/{self.lakehouse_path}"
+
+    @property
+    def local_path(self) -> Path:
+        """Return the local cache path corresponding to this lakehouse location."""
+        return CACHE_DIR / self.workspace / self.lakehouse / self.area / self.relative

binaryrain_fabric_local_pyspark-0.1.2/src/binaryrain_fabric_local_pyspark/spark.py

@@ -0,0 +1,166 @@
+"""Provide local Spark readers and SQL execution with Fabric-backed caching."""
+
+import hashlib
+import logging
+import os
+import subprocess
+from functools import partial
+from typing import Any
+
+from pyspark import SparkConf, SparkContext, java_gateway
+from pyspark.sql import DataFrame
+from pyspark.sql import SparkSession as OriginalSparkSession
+from pyspark.sql.readwriter import DataFrameReader as OriginalDataFrameReader
+
+from binaryrain_helper_local_pyspark.config import CACHE_DIR
+from binaryrain_helper_local_pyspark.fabric import resolve_local_path
+from binaryrain_helper_local_pyspark.sql import get_fabric_engine, get_fabric_table_from_lakehouse
+
+logger = logging.getLogger(__name__)
+
+
+class DataFrameReader(OriginalDataFrameReader):
+    """Resolve Fabric lakehouse parquet paths to local cache paths before reading."""
+
+    def parquet(self, *paths: str, **options: str | float | bool | None) -> DataFrame:
+        """
+        Read parquet data, downloading Fabric lakehouse paths to the cache when needed.
+
+        :param paths:
+            One or more abfss URIs, default lakehouse-relative paths, or local paths.
+        :param options:
+            Keyword options passed unchanged to PySpark's parquet reader.
+
+        :returns DataFrame:
+            DataFrame loaded from the resolved local paths.
+        exception : ValueError
+            If a default lakehouse-relative path is used without the required Fabric IDs.
+        """
+        return super().parquet(*(resolve_local_path(path) for path in paths), **options)
+
+
+class SparkSession(OriginalSparkSession):
+    """Use the caching reader and Fabric SQL endpoint for supported Spark operations."""
+
+    @property
+    def read(self) -> DataFrameReader:
+        """Return a reader that resolves Fabric lakehouse paths through the local cache."""
+        return DataFrameReader(self)
+
+    def sql(
+        self,
+        sqlQuery: str,
+        args: dict[str, Any] | list[Any] | None = None,
+        **kwargs: Any,
+    ) -> DataFrame:
+        """
+        Run a SQL query through the Fabric endpoint and cache its result, or use local Spark.
+
+        The cache key is a SHA-256 hash of the endpoint and the query text, so the same
+        query maps to the same parquet file. Non-empty ``args`` or ``kwargs`` bypass the
+        Fabric endpoint and cache, and are passed to the base Spark SQL implementation.
+
+        :param str sqlQuery:
+            T-SQL query to run through the Fabric SQL endpoint when no arguments are supplied.
+        :param dict[str, Any] | list[Any] | None args:
+            Spark SQL parameters. Non-empty values select local Spark execution. Default is None.
+        :param kwargs:
+            Keyword arguments passed to the base Spark SQL implementation. Non-empty values
+            select local Spark execution.
+
+        :returns DataFrame:
+            The query result, loaded from the cache for Fabric endpoint queries.
+        exception : ValueError
+            If no server is configured when the SQL engine is first created.
+        """
+        if args or kwargs:
+            return super().sql(sqlQuery, args, **kwargs)
+
+        server = os.getenv("FABRIC_SERVER", "")
+        cache_id = hashlib.sha256(f"{server}\n{sqlQuery.strip()}".encode()).hexdigest()
+        target = CACHE_DIR.parent / "sql" / f"{cache_id}.parquet"
+
+        if target.exists() and os.getenv("FABRIC_CACHE_REFRESH") != "1":
+            logger.info("Cache hit, reading local query result: %s", target)
+        else:
+            logger.info("Cache miss, querying Fabric SQL endpoint: %s", cache_id)
+            result = get_fabric_table_from_lakehouse(get_fabric_engine(), sqlQuery)
+            target.parent.mkdir(parents=True, exist_ok=True)
+            tmp_path = target.with_name(f"{target.name}.tmp")
+            # Spark cannot read nanosecond timestamps, which pandas writes by default.
+            result.to_parquet(
+                tmp_path,
+                index=False,
+                coerce_timestamps="us",
+                allow_truncated_timestamps=True,
+            )
+            tmp_path.replace(target)
+
+        return self.read.parquet(str(target))
+
+
+def get_local_spark() -> SparkSession:
+    """
+    Create or reuse a local Spark session with the Fabric-aware reader installed.
+
+    The PySpark builder creates its base session class. This function changes the resulting
+    session's class so its ``read`` property uses :class:`DataFrameReader`.
+
+    :returns SparkSession:
+        Local Spark session configured to use UTC and resolve Fabric data through the cache.
+    """
+    verbose = _spark_verbose()
+    conf = (
+        SparkConf()
+        .setMaster("local[*]")
+        .setAppName("local-fabric")
+        .set("spark.sql.session.timeZone", "UTC")
+        .set("spark.sql.debug.maxToStringFields", "200")
+        .set("spark.ui.showConsoleProgress", str(verbose).lower())
+    )
+    if not verbose:
+        _launch_quiet_gateway(conf)
+    spark = OriginalSparkSession.builder.config(conf=conf).getOrCreate()
+    spark.sparkContext.setLogLevel("INFO" if verbose else "ERROR")
+    spark.__class__ = SparkSession
+    return spark
+
+
+def _spark_verbose() -> bool:
+    """
+    Return whether Spark and JVM console output should be shown.
+
+    Verbose output is enabled when the package logger is at DEBUG level or
+    ``FABRIC_SPARK_DEBUG`` is set to ``1``.
+
+    :returns bool:
+        Whether verbose Spark output is wanted.
+    """
+    return logger.isEnabledFor(logging.DEBUG) or os.getenv("FABRIC_SPARK_DEBUG") == "1"
+
+
+def _launch_quiet_gateway(conf: SparkConf) -> None:
+    """
+    Start the Spark JVM with its console output redirected to a log file.
+
+    The JVM prints startup warnings before any log level can be applied, and PySpark
+    runs ``taskkill`` at exit on Windows, which prints to the console. Both are routed
+    away from the console here.
+
+    :param SparkConf conf:
+        Spark configuration used to launch the JVM.
+    """
+    if SparkContext._gateway is not None:
+        return
+    log_path = CACHE_DIR.parent / "spark.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    log_file = log_path.open("w", encoding="utf-8")
+    gateway = java_gateway.launch_gateway(
+        conf, popen_kwargs={"stdout": log_file, "stderr": subprocess.STDOUT}
+    )
+    SparkContext._gateway = gateway
+    SparkContext._jvm = gateway.jvm
+    java_gateway.Popen = partial(
+        subprocess.Popen, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+    )
+    logger.debug("Spark JVM output is written to %s", log_path)

binaryrain_fabric_local_pyspark-0.1.2/src/binaryrain_fabric_local_pyspark/sql.py

@@ -0,0 +1,81 @@
+"""Connect to a Fabric lakehouse SQL endpoint using Azure token authentication."""
+
+import os
+import struct
+from functools import cache
+from urllib.parse import quote_plus
+
+import pandas as pd
+from azure.identity import DefaultAzureCredential
+from sqlalchemy import create_engine, event
+from sqlalchemy.engine import Engine
+
+from binaryrain_helper_local_pyspark.config import FABRIC_TOKEN_SCOPE, SQL_COPT_SS_ACCESS_TOKEN
+
+
+@cache
+def get_fabric_engine(
+    server: str | None = None,
+    database: str | None = None,
+) -> Engine:
+    """Create a cached SQLAlchemy engine for a Fabric lakehouse SQL endpoint.
+
+    Each new physical connection receives a fresh Azure Active Directory access token.
+    ``server`` and ``database`` override ``FABRIC_SERVER`` and ``FABRIC_DATABASE``.
+
+    :param str | None server:
+        Fabric SQL endpoint host. Defaults to ``FABRIC_SERVER``.
+    :param str | None database:
+        Lakehouse database name. Defaults to ``FABRIC_DATABASE``, then ``default``.
+
+    :returns Engine:
+        SQLAlchemy engine configured for ODBC Driver 18 and Azure token authentication.
+    exception : ValueError
+        If neither ``server`` nor ``FABRIC_SERVER`` provides a server name.
+    """
+
+    fabric_server = server or os.getenv("FABRIC_SERVER")
+    database = database or os.getenv("FABRIC_DATABASE", "default")
+    if not fabric_server:
+        raise ValueError("FABRIC_SERVER environment variables must be set.")
+
+    connection_string = (
+        f"Driver={{ODBC Driver 18 for SQL Server}};"
+        f"Server={fabric_server};"
+        f"Database={database or 'default'};"
+        f"Encrypt=yes;"
+        f"TrustServerCertificate=no;"
+    )
+    url = f"mssql+pyodbc:///?odbc_connect={quote_plus(connection_string)}"
+    engine = create_engine(url)
+
+    credential = DefaultAzureCredential()
+
+    @event.listens_for(engine, "do_connect")
+    def provide_token(dialect, conn_rec, cargs, cparams):
+        """Set the Azure access token on the ODBC connection parameters."""
+        token = credential.get_token(FABRIC_TOKEN_SCOPE).token.encode("utf-16-le")
+        token_struct = struct.pack(f"<I{len(token)}s", len(token), token)
+        cparams["attrs_before"] = {SQL_COPT_SS_ACCESS_TOKEN: token_struct}
+
+    return engine
+
+
+def get_fabric_table_from_lakehouse(
+    engine: Engine,
+    query: str,
+) -> pd.DataFrame:
+    """Execute a SQL query against Fabric and return its rows as a DataFrame.
+
+    :param Engine engine:
+        SQLAlchemy engine connected to the Fabric lakehouse endpoint.
+    :param str query:
+        SQL query to execute.
+
+    :returns pd.DataFrame:
+        Rows returned by the query, with columns named from the result set.
+    """
+    result = pd.DataFrame()
+    with engine.connect() as conn:
+        result = pd.read_sql(query, conn)
+    return result