adapta 2.11.9__py3-none-any.whl → 3.5.13__py3-none-any.whl

adapta/storage/database/v3/odbc.py

@@ -0,0 +1,217 @@
+"""
+ Database client that uses an ODBC driver.
+"""
+
+#  Copyright (c) 2023-2026. ECCO Data & AI and other project contributors.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from abc import ABC
+from typing import Optional
+from collections.abc import Iterator
+
+from pandas import read_sql
+import sqlalchemy
+from sqlalchemy import text
+from sqlalchemy.connectors import pyodbc
+from sqlalchemy.engine import URL
+from sqlalchemy.exc import SQLAlchemyError, OperationalError
+
+from adapta.logs import SemanticLogger
+from adapta.storage.database.v3.models import DatabaseType, SqlAlchemyDialect
+from adapta.utils.metaframe import MetaFrame
+
+
+class OdbcClient(ABC):
+    """
+    Generic ODBC database client that relies on SqlAlchemy API.
+    """
+
+    def __init__(
+        self,
+        logger: SemanticLogger,
+        database_type: DatabaseType,
+        host_name: str | None = None,
+        user_name: str | None = None,
+        database: str | None = None,
+        password: str | None = None,
+        port: int | None = None,
+    ):
+        """
+         Creates an instance of an OdbcClient
+
+        :param logger: Logger instance for database operations.
+        :param database_type: Type of database to connect to.
+        :param host_name: Host name.
+        :param user_name: SQL user name.
+        :param database: Optional database name to connect to.
+        :param password: SQL user password.
+        :param port: Connection port.
+        """
+        self._db_type = database_type
+        self._dialect: SqlAlchemyDialect = database_type.value
+        self._host = host_name
+        self._database = database
+        self._user = user_name
+        self._password = password
+        self._port = port
+        self._logger = logger
+        self._engine = None
+        self._connection = None
+        pyodbc.pooling = False
+
+    def __enter__(self) -> Optional["OdbcClient"]:
+        connection_url: sqlalchemy.engine.URL = URL.create(
+            drivername=self._dialect.dialect,
+            host=self._host,
+            database=self._database,
+            username=self._user,
+            password=self._password,
+            port=self._port,
+            query=self._dialect.driver,
+        )
+        self._logger.info(
+            "Connecting to {host}:{port} using dialect {dialect} and driver {driver}",
+            host=self._host,
+            port=self._port,
+            dialect=self._dialect.dialect,
+            driver=self._dialect.driver,
+        )
+        try:
+            self._engine: sqlalchemy.engine.Engine = sqlalchemy.create_engine(connection_url, pool_pre_ping=True)
+            self._connection: sqlalchemy.engine.Connection = self._engine.connect()
+            return self
+        except SQLAlchemyError as ex:
+            self._logger.error(
+                "Error connecting to {host}:{port} using dialect {dialect} and driver {driver}",
+                host=self._host,
+                port=self._port,
+                dialect=self._dialect.dialect,
+                driver=self._dialect.driver,
+                exception=ex,
+            )
+
+            return None
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._connection.close()
+        self._engine.dispose()
+
+    def fork(self) -> "OdbcClient":
+        """
+        Copies this client in order to create a new connection, while keeping the other one open (fork).
+        """
+        return OdbcClient(
+            logger=self._logger,
+            database_type=self._db_type,
+            host_name=self._host,
+            user_name=self._user,
+            password=self._password,
+            port=self._port,
+        )
+
+    def _get_connection(self) -> sqlalchemy.engine.Connection | None:
+        if self._connection is None:
+            self._logger.info("No connection is active. Please create one using with OdbcClient(..) as client: ...")
+            return None
+
+        return self._connection
+
+    def query(self, query: str, chunksize: int | None = None) -> MetaFrame | Iterator[MetaFrame] | None:
+        """
+          Read result of SQL query into a MetaFrame. The latent representation of the MetaFrame is a Pandas dataframe.
+
+        :param query: Query to execute on the connection.
+        :param chunksize: Size of an individual data chunk. If not provided, query result will be a single dataframe.
+        :return:
+        """
+        try:
+            if chunksize:
+                return (
+                    MetaFrame.from_pandas(chunk)
+                    for chunk in read_sql(query, con=self._get_connection(), chunksize=chunksize)
+                )
+
+            return MetaFrame.from_pandas(read_sql(query, con=self._get_connection()))
+        except SQLAlchemyError as ex:
+            self._logger.error("Engine error while executing query {query}", query=query, exception=ex)
+            return None
+        except BaseException as other:  # pylint: disable=W0703
+            self._logger.error(
+                "Unknown error while executing query {query}",
+                query=query,
+                exception=other,
+            )
+            return None
+
+    def materialize(
+        self,
+        data: MetaFrame,
+        schema: str,
+        name: str,
+        overwrite: bool = False,
+        chunksize: int | None = None,
+    ) -> int | None:
+        """
+          Materialize MetaFrame as a table in a database.
+          The table is converted to a Pandas dataframe before materialization.
+
+        :param data: MetaFrame to materialize as a table.
+        :param schema: Schema of a table.
+        :param name: Name of a table.
+        :param overwrite: Whether to overwrite or append the data.
+        :param chunksize: Use this to split a dataframe into chunks and append them sequentially to the target table.
+        :return:
+        """
+        try:
+            if overwrite:
+                try:
+                    if self._dialect.dialect == DatabaseType.SQLITE_ODBC.value.dialect:
+                        self._get_connection().execute(text(f"DELETE FROM {schema}.{name}"))
+                    else:
+                        self._get_connection().execute(text(f"TRUNCATE TABLE {schema}.{name}"))
+                except OperationalError as ex:
+                    # The table does not exist. Do nothing and let the Pandas API handle the creation of the table.
+                    self._logger.warning(
+                        "Error truncating {schema}.{table}, now creating table without truncating.",
+                        schema=schema,
+                        table=name,
+                        exception=ex,
+                    )
+
+            return data.to_pandas().to_sql(
+                name=name,
+                schema=schema,
+                con=self._get_connection(),
+                index=False,
+                chunksize=chunksize,
+                if_exists="append",
+            )
+        except SQLAlchemyError as ex:
+            self._logger.error(
+                "Error while materializing a dataframe into {schema}.{table}",
+                schema=schema,
+                table=name,
+                exception=ex,
+            )
+            return None
+        finally:
+            active_tran: sqlalchemy.engine.RootTransaction = self._get_connection().get_transaction()
+            if active_tran and active_tran.is_active:
+                self._logger.debug(
+                    "Found an active transaction for {schema}.{table}. Committing it.",
+                    schema=schema,
+                    table=name,
+                )
+                active_tran.commit()

adapta/storage/database/v3/snowflake_sql.py

@@ -0,0 +1,241 @@
+"""
+  Snowflake Client Wrapper
+"""
+
+import os
+import re
+from types import TracebackType
+from typing import Self
+
+import snowflake.connector
+
+from snowflake.connector.errors import DatabaseError, ProgrammingError
+
+from adapta.logs.models import LogLevel
+from adapta.logs import SemanticLogger
+from adapta.storage.models import S3Path, DataPath
+
+from adapta.storage.models.azure import AdlsGen2Path
+from adapta.utils.metaframe import MetaFrame
+
+
+class SnowflakeClient:
+    """
+    A wrapper around the Snowflake Python connector that provides a context manager for handling connections
+    and transactions. It also includes a method for executing queries and returning the result as a Pandas DataFrame.
+
+    :param user: The username for the Snowflake account.
+    :param account: The account name for the Snowflake account.
+    :param warehouse: The warehouse name for the Snowflake account.
+    :param authenticator: The authentication mechanism to use for the Snowflake account.
+    :param logger: The logger to use for logging messages. Defaults to a new SemanticLogger instance.
+    :param password: Optional - The password for the Snowflake user. Should be combined with `authenticator='snowflake'` to enable password authentication
+    :param role: Optional - The role for the Snowflake user.
+    """
+
+    def __init__(
+        self,
+        user: str,
+        account: str,
+        warehouse: str,
+        authenticator: str = "externalbrowser",
+        logger: SemanticLogger = SemanticLogger().add_log_source(
+            log_source_name="adapta-snowflake-client",
+            min_log_level=LogLevel.INFO,
+            is_default=True,
+        ),
+        password: str | None = None,
+        role: str | None = None,
+    ):
+        self._user = user
+        self._account = account
+        self._warehouse = warehouse
+        self._authenticator = "snowflake" if password else authenticator
+        self._logger = logger
+        self._password = password
+        self._role = role
+        self._conn = None
+
+    def __enter__(self) -> Self | None:
+        """
+        Enters the context manager and establishes a connection to the Snowflake database.
+        :return: The SnowflakeClient instance, or None if there was an error connecting to the database.
+        """
+        try:
+            self._conn = snowflake.connector.connect(
+                user=self._user,
+                account=self._account,
+                password=self._password,
+                warehouse=self._warehouse,
+                authenticator=self._authenticator,
+                role=self._role,
+            )
+            return self
+        except DatabaseError as ex:
+            self._logger.error(
+                "Error connecting to {account} for {user}", account=self._account, user=self._user, exception=ex
+            )
+            return None
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None = None,
+        exc_val: BaseException | None = None,
+        exc_tb: TracebackType | None = None,
+    ) -> None:
+        """
+        Exits the context manager and closes the database connection.
+
+        :param exc_type: The type of the exception that was raised, if any.
+        :param exc_val: The value of the exception that was raised, if any.
+        :param exc_tb: The traceback of the exception that was raised, if any.
+        """
+        self._conn.close()
+        if exc_val is not None:
+            self._logger.error(f"An error occurred while closing the database connection: {exc_val}")
+
+    def query(self, query: str, fetch_dataframe: bool = True) -> MetaFrame | None:
+        """
+        Executes the given SQL query and returns the result as a Pandas DataFrame.
+
+        :param query: The SQL query to execute.
+        :param fetch_dataframe: Fetch dataframes in batches, otherwise only execute the query
+        :return: An iterator of Pandas DataFrames, one for each result set returned by the query, or None if there was
+            an error executing the query.
+        """
+        try:
+            with self._conn.cursor() as cursor:
+                result = cursor.execute(query)
+                if fetch_dataframe:
+                    return MetaFrame.from_arrow(result.fetch_arrow_all(force_return_table=True))
+                return None
+
+        except ProgrammingError as ex:
+            self._logger.error("Error executing query {query}", query=query, exception=ex)
+            return None
+
+    def _get_snowflake_type(self, data_type: str) -> str:
+        """Maps delta type to Snowflake type"""
+
+        type_map = {
+            "string": "TEXT",
+            "integer": "INTEGER",
+            "float": "FLOAT",
+            "double": "FLOAT",
+            "timestamp": "TIMESTAMP_NTZ",
+            "date": "DATE",
+            "struct": "VARIANT",
+            "map": "VARIANT",
+            "array": "VARIANT",
+            "boolean": "BOOLEAN",
+            "binary": "BINARY",
+        }
+
+        snowflake_type = type_map.get(data_type, None)
+        if snowflake_type:
+            return snowflake_type
+
+        if data_type.startswith("decimal"):
+            decimal_info = [int(num) for num in re.findall(r"\d+", data_type)]
+            return f"DECIMAL({decimal_info[0]},{decimal_info[1]})"
+
+        raise ValueError(f"found type:{data_type} which is currently not supported")
+
+    def publish_external_delta_table(
+        self,
+        database: str,
+        schema: str,
+        table: str,
+        refresh_metadata_only: bool = False,
+        path: DataPath | None = None,
+        table_schema: dict[str, str] | None = None,
+        partition_columns: list[str] | None = None,
+        storage_integration: str | None = None,
+    ) -> None:
+        """
+        Creates delta table as external table in Snowflake
+
+        :param database: name of the database, in Snowflake, to create the table
+        :param schema: name of the schema, in Snowflake, to create the table
+        :param table: name of the table to be created in Snowflake
+        :param refresh_metadata_only: Only refresh metadata, when table has already existed in snowflake.
+                                      So skip the initializing phases like creating schema, creating external table, etc.
+        :param path: path to the delta table in datalake
+        :param table_schema: A mapping from column name to column type (the type should be in the lower case and supported by delta table)
+                             , like {'ColumnA': 'struct', 'ColumnB': 'decimal(10, 2)'}
+        :param partition_columns: A list of partition column names
+        :param storage_integration: name of the storage integration to use in Snowflake. Default to the name of the storage account
+        """
+
+        def _get_azure_query(resolved_path: AdlsGen2Path):
+            return (
+                f"create stage if not exists {database}.{schema}.stage_{table}"
+                + f" storage_integration = {storage_integration if storage_integration is not None else resolved_path.account}"
+                + f" url = 'azure://{resolved_path.account}.blob.core.windows.net/{resolved_path.container}/{path.path}';"
+            )
+
+        def _get_s3_query(resolved_path: S3Path):
+            return (
+                f"create stage if not exists {database}.{schema}.stage_{table}"
+                + f" endpoint = '{os.environ['S3_ENDPOINT__DNS_NAME']}'"
+                + f" url = 's3compat://{resolved_path.bucket}/{resolved_path.path}'"
+                + f" credentials = (AWS_KEY_ID = '{os.environ['S3_ENDPOINT__ACCESS_KEY_ID']}' AWS_SECRET_KEY = '{os.environ['S3_ENDPOINT__ACCESS_KEY']}');"
+            )
+
+        if not refresh_metadata_only:
+            assert path, "Path to the delta table needed! Please check!"
+            assert table_schema, "Table schema needed! Please check!"
+
+            self.query(query=f"create schema if not exists {database}.{schema}", fetch_dataframe=False)
+
+            if isinstance(path, AdlsGen2Path):
+                query = _get_azure_query(path)
+            elif isinstance(path, S3Path):
+                query = _get_s3_query(path)
+            else:
+                raise ValueError(f"Path type {type(path)} is not supported!")
+
+            self.query(
+                query=query,
+                fetch_dataframe=False,
+            )
+
+            if partition_columns is not None:
+                partition_expr = ",".join(partition_columns)
+                partition_select = [
+                    f"\"{partition_column}\" TEXT AS (split_part(split_part(metadata$filename, '=', {2 + i}), '/', 1))"
+                    for i, partition_column in enumerate(partition_columns)
+                ]
+            else:
+                partition_expr = ""
+                partition_select = []
+                partition_columns = []
+
+            snowflake_columns = [
+                (column_name, self._get_snowflake_type(column_type))
+                for column_name, column_type in table_schema.items()
+                if column_name not in partition_columns
+            ]
+
+            columns = [
+                f'"{column}" {col_type} AS ($1:"{column}"::{col_type})' for column, col_type in snowflake_columns
+            ] + partition_select
+
+            column_expr = ("," + os.linesep).join(columns)
+
+            self.query(
+                query=f"""
+                create or replace external table "{database}"."{schema}"."{table}"
+                (
+                    {column_expr}
+                )
+                {f"partition by ({partition_expr})" if partition_expr else ""}
+                location=@{database}.{schema}.stage_{table}  
+                auto_refresh = false   
+                refresh_on_create=false   
+                file_format = (type = parquet)    
+                table_format = delta;""",
+                fetch_dataframe=False,
+            )
+
+        self.query(query=f'alter external table "{database}"."{schema}"."{table}" refresh;', fetch_dataframe=False)

adapta/storage/database/v3/trino_sql.py

@@ -0,0 +1,154 @@
+"""
+  SqlAlchemy-based Trino Client Wrapper
+"""
+
+#  Copyright (c) 2023-2026. ECCO Data & AI and other project contributors.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import os
+from dataclasses import dataclass
+from typing import final, Optional
+from collections.abc import Iterator
+
+import sqlalchemy
+from pandas import read_sql_query
+from sqlalchemy import create_engine
+from sqlalchemy.exc import SQLAlchemyError
+from trino.auth import OAuth2Authentication, BasicAuthentication
+
+from adapta.logs.models import LogLevel
+from adapta.logs import SemanticLogger
+from adapta.storage.secrets import SecretStorageClient
+from adapta.utils.metaframe import MetaFrame
+
+
+@final
+@dataclass
+class TrinoConnectionSecret:
+    """
+    Connection secret structure for Trino
+    """
+
+    secret_name: str
+    username_secret_key: str
+    password_secret_key: str
+
+
+class TrinoClient:
+    """
+    Trino (https://www.trino.io) connection client.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        catalog: str | None = None,
+        port: int | None = 443,
+        oauth2_username: str | None = None,
+        credentials_provider: tuple[TrinoConnectionSecret, SecretStorageClient] | None = None,
+        logger: SemanticLogger = SemanticLogger().add_log_source(
+            log_source_name="adapta-trino-client",
+            min_log_level=LogLevel.INFO,
+            is_default=True,
+        ),
+    ):
+        """
+         Initializes a SqlAlchemy Engine that will facilitate connections to Trino.
+         Authentication options:
+          - via OAuth2 if oauth2_username or ADAPTA__TRINO_OAUTH2_USERNAME is provided
+          - via external secret provider (Vault, Azure KeyVault, AWS Secrets Manager, etc.) if credentials_provider is provided
+          - via plaintext username-password if ADAPTA__TRINO_USERNAME and ADAPTA__TRINO_PASSWORD are provided
+
+        :param host: Trino Coordinator hostname, without protocol.
+        :param catalog: Trino catalog.
+        :param port: Trino connection port (443 default).
+        :param oauth2_username: Optional username to use if authenticating with interactive OAuth2.
+               Can also be provided via ADAPTA__TRINO_OAUTH2_USERNAME.
+        :param credentials_provider: Optional secret provider and auth secret details to use to read Basic Auth credentials.
+        :param logger: CompositeLogger instance.
+        """
+
+        self._host = host
+        self._catalog = catalog
+        self._port = port
+        if "ADAPTA__TRINO_USERNAME" in os.environ:
+            self._engine = create_engine(
+                f"trino://{os.getenv('ADAPTA__TRINO_USERNAME')}@{self._host}:{self._port}/{self._catalog or ''}",
+                connect_args={
+                    "auth": BasicAuthentication(
+                        os.getenv("ADAPTA__TRINO_USERNAME"), os.getenv("ADAPTA__TRINO_PASSWORD")
+                    ),
+                    "http_scheme": "https",
+                },
+            )
+        elif "ADAPTA__TRINO_OAUTH2_USERNAME" in os.environ or oauth2_username:
+            self._engine = create_engine(
+                f"trino://{os.getenv('ADAPTA__TRINO_OAUTH2_USERNAME')}@{self._host}:{self._port}/{self._catalog or ''}",
+                connect_args={
+                    "auth": OAuth2Authentication(),
+                    "http_scheme": "https",
+                },
+            )
+        elif credentials_provider:
+            credentials_secret = credentials_provider[1].read_secret("", credentials_provider[0].secret_name)
+            username = credentials_secret[credentials_provider[0].username_secret_key]
+            self._engine = create_engine(
+                f"trino://{username}@{self._host}:{self._port}/{self._catalog or ''}",
+                connect_args={
+                    "auth": BasicAuthentication(
+                        username, credentials_secret[credentials_provider[0].password_secret_key]
+                    ),
+                    "http_scheme": "https",
+                },
+            )
+        else:
+            raise ConnectionError(
+                "Neither ADAPTA__TRINO_USERNAME or ADAPTA__TRINO_OAUTH2_USERNAME is specified. Cannot authenticate to the provided host."
+            )
+
+        self._logger = logger
+        self._connection: sqlalchemy.engine.Connection | None = None
+
+    def __enter__(self) -> Optional["TrinoClient"]:
+        try:
+            self._connection = self._engine.connect()
+            return self
+        except SQLAlchemyError as ex:
+            self._logger.error(
+                "Error connecting to {host}:{port}",
+                host=self._host,
+                port=self._port,
+                exception=ex,
+            )
+            return None
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._connection.close()
+        self._engine.dispose()
+
+    def query(self, query: str, batch_size: int = 1000) -> Iterator[MetaFrame]:
+        """
+        Executes a Trino DML query and converts the result into a Pandas dataframe.
+
+        This method internally calls pandas.read_sql_query
+
+        :param query: SQL query compliant with https://trino.io/docs/current/sql.html
+        :param batch_size: Optional batch size to return rows iteratively.
+        """
+
+        return (
+            MetaFrame.from_pandas(chunk)
+            for chunk in read_sql_query(sql=query, con=self._connection, chunksize=batch_size)
+        )

adapta/storage/delta_lake/__init__.py

@@ -1,7 +1,7 @@
 """
  Import index.
 """
-#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#  Copyright (c) 2023-2026. ECCO Data & AI and other project contributors.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -16,5 +16,4 @@
 #  limitations under the License.
 #
 
-from adapta.storage.delta_lake._functions import *
-from adapta.storage.delta_lake._models import *
+from adapta.storage.delta_lake.v2 import *

adapta/storage/delta_lake/{README.md → v2/README.md}

@@ -1,5 +1,7 @@
 # Delta Lake Operations
 
+**This is a deprecated module. Please use the new module `adapta.storage.delta_lake.v3` instead.**
+
 Supported API:
 - read delta table as `pandas.DataFrame`
 - read delta table in batches of a provided size, each batch being `pandas.DataFrame`

adapta/storage/delta_lake/v2/__init__.py

@@ -0,0 +1,19 @@
+"""
+Import index
+"""
+#  Copyright (c) 2023-2026. ECCO Data & AI and other project contributors.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from adapta.storage.delta_lake.v2._models import *
+from adapta.storage.delta_lake.v2._functions import *