ofwat-dataprepkit 0.4__tar.gz

ofwat_dataprepkit-0.4/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Ofwat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ofwat_dataprepkit-0.4/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: ofwat-dataprepkit
+Version: 0.4
+Summary: ETL Tools for Fabric
+Home-page: https://github.com/Ofwat/dataprepkit
+Author: Ofwat
+Project-URL: Source, https://github.com/Ofwat/dataprepkit
+Project-URL: Tracker, https://github.com/Ofwat/dataprepkit/issues
+Keywords: fabric etl
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: sqlalchemy
+Requires-Dist: pyodbc
+Dynamic: author
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: summary
+
+# dataprepkit
+A collection of helpers and utilities for SQL-based data ingestion, transformation, and pipeline orchestration.

ofwat_dataprepkit-0.4/README.md

@@ -0,0 +1,2 @@
+# dataprepkit
+A collection of helpers and utilities for SQL-based data ingestion, transformation, and pipeline orchestration.

ofwat_dataprepkit-0.4/dataprepkit/__init__.py

@@ -0,0 +1,2 @@
+"""Main entry point into the dataprepkit package
+"""

ofwat_dataprepkit-0.4/dataprepkit/helpers/connectors/warehouse.py

@@ -0,0 +1,140 @@
+"""
+Fabric Warehouse SQLAlchemy Engine Utilities
+
+This module provides utility functions to create and validate SQLAlchemy engine
+connections to Azure Fabric data warehouses using Azure Active Directory (AAD) tokens
+for authentication. It handles automatic detection of the latest suitable ODBC driver
+and integrates with Fabric's `notebookutils.credentials` (or mock credentials for local testing).
+
+Functions:
+    - get_fabric_warehouse_engine: Create a SQLAlchemy engine for a given Fabric SQL endpoint.
+    - validate_fabric_warehouse_engine: Run a test query to verify the engine connection.
+
+Dependencies:
+    - pyodbc
+    - sqlalchemy
+    - notebookutils (optional, Fabric environment)
+"""
+
+import logging as _logging
+import re as _re
+import struct as _struct
+import pyodbc as _pyodbc
+import sqlalchemy as _sa
+
+_logger = _logging.getLogger(__name__)
+
+# Explicitly define public API
+__all__ = [
+    "get_fabric_warehouse_engine",
+    "validate_fabric_warehouse_engine"
+]
+
+# -----------------------------
+# Handle Fabric-only dependency
+# -----------------------------
+try:
+    from notebookutils import credentials as _default_credentials # type: ignore
+except ImportError:
+    # pylint: disable=R0903
+    class _MockCredentials:
+        # pylint: disable=C0103
+        def getToken(self, resource: str) -> str:
+            """Mocking getToken from mssparkutls"""
+            _logger.warning("Using mock credentials for resource: %s", resource)
+            return "FAKE_TOKEN_FOR_LOCAL_TESTING"
+    _default_credentials = _MockCredentials()
+
+
+def _get_latest_sql_driver() -> str:
+    drivers = _pyodbc.drivers() # pylint: disable=I1101
+    sql_drivers = [d for d in drivers if "SQL Server" in d or "ODBC Driver" in d]
+    if not sql_drivers:
+        raise RuntimeError("No suitable ODBC driver for SQL Server found.")
+
+    def extract_version(name: str) -> int:
+        match = _re.search(r"(\d+)", name)
+        return int(match.group(1)) if match else 0
+
+    latest_driver = max(sql_drivers, key=extract_version)
+    _logger.info("Using ODBC driver: %s", latest_driver)
+    return latest_driver
+
+
+def get_fabric_warehouse_engine(
+        sql_endpoint: str,
+        port: int = 1433,
+        credentials=_default_credentials
+    ) -> _sa.engine.Engine:
+    # pylint: disable=C0301
+    """
+    Create and return a SQLAlchemy engine connected to an Azure Fabric data warehouse.
+
+    Args:
+        sql_endpoint (str): The Fabric SQL endpoint to connect to.
+        port (int, optional): The TCP port for the SQL server. Defaults to 1433.
+        credentials (optional): An object with a getToken(resource) method. Defaults to Fabric's credentials.
+
+    Returns:
+        _sa.engine.Engine: A SQLAlchemy Engine instance connected to the Fabric warehouse.
+
+    Raises:
+        ValueError: If `sql_endpoint` is empty or None.
+        RuntimeError: If no suitable ODBC driver is found.
+        Exception: If token retrieval or engine creation fails.
+    """
+    if not sql_endpoint:
+        raise ValueError("sql_endpoint is required and cannot be empty.")
+
+    try:
+        driver = _get_latest_sql_driver()
+        server = f"{sql_endpoint},{port}"
+
+        token = credentials.getToken("https://database.windows.net/").encode("UTF-16-LE")
+        token_struct = _struct.pack(f"<I{len(token)}s", len(token), token)
+
+        connection_string = f"DRIVER={{{driver}}};SERVER={server}"
+        connection_url = _sa.engine.URL.create(
+            "mssql+pyodbc",
+            query={"odbc_connect": connection_string}
+        )
+
+        engine = _sa.create_engine(
+            connection_url,
+            connect_args={"attrs_before": {1256: token_struct}},
+            pool_pre_ping=True,
+            pool_recycle=3600
+        )
+
+        _logger.info("Successfully created Fabric SQL engine.")
+        return engine
+
+    except Exception as ex:
+        _logger.error("Failed to create Fabric engine: %s", ex, exc_info=True)
+        raise
+
+
+def validate_fabric_warehouse_engine(engine: _sa.engine.Engine) -> bool:
+    """
+    Test the connection to the Fabric warehouse engine by executing a simple query.
+
+    Args:
+        engine (_sa.engine.Engine): The SQLAlchemy engine connected to the Fabric warehouse.
+
+    Returns:
+        bool: True if the test query executes successfully and returns the expected result.
+
+    Raises:
+        Exception: If the query fails to execute or returns an unexpected result.
+    """
+    try:
+        with engine.connect() as conn:
+            result = conn.execute(_sa.text("SELECT 1"))
+            value = result.scalar()
+            if value == 1:
+                _logger.info("Fabric warehouse engine test passed.")
+                return True
+            raise RuntimeError("Unexpected result from test query.")
+    except Exception as ex:
+        _logger.error("Test query failed: %s", ex, exc_info=True)
+        raise

ofwat_dataprepkit-0.4/ofwat_dataprepkit.egg-info/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: ofwat-dataprepkit
+Version: 0.4
+Summary: ETL Tools for Fabric
+Home-page: https://github.com/Ofwat/dataprepkit
+Author: Ofwat
+Project-URL: Source, https://github.com/Ofwat/dataprepkit
+Project-URL: Tracker, https://github.com/Ofwat/dataprepkit/issues
+Keywords: fabric etl
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: sqlalchemy
+Requires-Dist: pyodbc
+Dynamic: author
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: summary
+
+# dataprepkit
+A collection of helpers and utilities for SQL-based data ingestion, transformation, and pipeline orchestration.

ofwat_dataprepkit-0.4/ofwat_dataprepkit.egg-info/SOURCES.txt

@@ -0,0 +1,26 @@
+LICENSE
+README.md
+setup.py
+dataprepkit/__init__.py
+dataprepkit/helpers.py
+dataprepkit/helpers/__init__.py
+dataprepkit/helpers/connectors/__init__.py
+dataprepkit/helpers/connectors/warehouse.py
+dataprepkit/helpers/connectors/lakehouse/__init__.py
+dataprepkit/helpers/connectors/lakehouse/lakehouse.py
+dataprepkit/helpers/dims/__init__.py
+dataprepkit/helpers/dims/dimension_registry.py
+dataprepkit/helpers/transforms/__init__.py
+dataprepkit/processors/__init__.py
+dataprepkit/processors/dimensions/__init__.py
+dataprepkit/processors/dimensions/dim_common.py
+dataprepkit/processors/dimensions/dim_company.py
+dataprepkit/processors/dimensions/dim_measure.py
+dataprepkit/processors/dimensions/dim_region.py
+dataprepkit/processors/quarterly/__init__.py
+dataprepkit/processors/quarterly/qdata_processor.py
+ofwat_dataprepkit.egg-info/PKG-INFO
+ofwat_dataprepkit.egg-info/SOURCES.txt
+ofwat_dataprepkit.egg-info/dependency_links.txt
+ofwat_dataprepkit.egg-info/requires.txt
+ofwat_dataprepkit.egg-info/top_level.txt

ofwat_dataprepkit-0.4/ofwat_dataprepkit.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

ofwat_dataprepkit-0.4/ofwat_dataprepkit.egg-info/requires.txt

@@ -0,0 +1,4 @@
+pandas
+numpy
+sqlalchemy
+pyodbc

ofwat_dataprepkit-0.4/ofwat_dataprepkit.egg-info/top_level.txt

@@ -0,0 +1 @@
+dataprepkit

ofwat_dataprepkit-0.4/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ofwat_dataprepkit-0.4/setup.py

@@ -0,0 +1,29 @@
+from setuptools import setup, find_packages
+
+# read the contents of your README file
+from os import path
+this_directory = path.abspath(path.dirname(__file__))
+with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
+setup(
+    name="ofwat-dataprepkit",
+    version="0.4",
+    author="Ofwat",
+    description='ETL Tools for Fabric',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    keywords="fabric etl",
+    packages=find_packages(),
+    url="https://github.com/Ofwat/dataprepkit",
+    project_urls={
+        'Source': 'https://github.com/Ofwat/dataprepkit',
+        'Tracker': 'https://github.com/Ofwat/dataprepkit/issues',
+    },
+    install_requires=[
+        "pandas",
+        "numpy",
+        "sqlalchemy",
+        "pyodbc",
+    ],
+)