adapta 2.11.9__py3-none-any.whl → 3.5.13__py3-none-any.whl

adapta/storage/delta_lake/{_functions.py → v2/_functions.py}

@@ -1,7 +1,8 @@
+# pylint: disable=duplicate-code
 """
  Operations on Delta Lake tables.
 """
-#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#  Copyright (c) 2023-2026. ECCO Data & AI and other project contributors.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -19,7 +20,8 @@
 import datetime
 import hashlib
 import zlib
-from typing import Optional, Union, Iterator, List, Iterable, Tuple
+from collections.abc import Iterator, Iterable
+from warnings import warn
 
 from pandas import DataFrame, concat
 import pyarrow
@@ -30,27 +32,29 @@ from pyarrow._dataset_parquet import ParquetReadOptions  # pylint: disable=E0611
 from adapta.logs import SemanticLogger
 from adapta.security.clients._base import AuthenticationClient
 from adapta.storage.models.base import DataPath
-from adapta.storage.delta_lake._models import DeltaTransaction
+from adapta.storage.delta_lake.v2._models import DeltaTransaction
 from adapta.storage.cache import KeyValueCache
-from adapta.storage.models.format import DataFrameParquetSerializationFormat
+from adapta.storage.models.formatters import PandasDataFrameParquetSerializationFormat
 from adapta.storage.models.filter_expression import Expression, ArrowFilterExpression, compile_expression
 
 
 def load(  # pylint: disable=R0913
     auth_client: AuthenticationClient,
     path: DataPath,
-    version: Optional[int] = None,
-    row_filter: Optional[Union[Expression, pyarrow.compute.Expression]] = None,
-    columns: Optional[List[str]] = None,
-    batch_size: Optional[int] = None,
-    partition_filter_expressions: Optional[List[Tuple]] = None,
-) -> Union[DeltaTable, DataFrame, Iterator[DataFrame]]:
+    version: int | None = None,
+    timestamp: datetime.datetime | None = None,
+    row_filter: Expression | pyarrow.compute.Expression | None = None,
+    columns: list[str] | None = None,
+    batch_size: int | None = None,
+    partition_filter_expressions: list[tuple] | None = None,
+) -> DeltaTable | DataFrame | Iterator[DataFrame]:
     """
      Loads Delta Lake table from Azure or AWS storage and converts it to a pandas dataframe.
 
     :param auth_client: AuthenticationClient for target storage.
     :param path: Path to delta table, in HDFS format: abfss://container@account.dfs.core.windows.net/my/path
-    :param version: Optional version to read. Defaults to latest.
+    :param version: Optional version to read. Defaults to latest. If set, timestamp will be ignored.
+    :param timestamp: Optional time travel timestamp. Allows to read data as of a specific time. Ignored if version is set.
     :param row_filter: Optional filter to apply, as pyarrow expression. Example:
       from pyarrow.dataset import field as pyarrow_field
 
@@ -66,9 +70,21 @@ def load(  # pylint: disable=R0913
 
     :return: A DeltaTable wrapped Rust class, pandas Dataframe or an iterator of pandas Dataframes, for batched reads.
     """
-    pyarrow_ds = DeltaTable(
-        path.to_delta_rs_path(), version=version, storage_options=auth_client.connect_storage(path)
-    ).to_pyarrow_dataset(
+    warn(
+        "You are using version 2 of the load function. "
+        "This is deprecated and will be removed in adapta version 4. "
+        "Please upgrade to version 3: adapta.storage.delta_lake.v3",
+        DeprecationWarning,
+    )
+    if version:
+        timestamp = None
+
+    pyarrow_ds = DeltaTable(path.to_delta_rs_path(), version=version, storage_options=auth_client.connect_storage(path))
+
+    if timestamp:
+        pyarrow_ds.load_as_version(timestamp)
+
+    pyarrow_ds = pyarrow_ds.to_pyarrow_dataset(
         partitions=partition_filter_expressions,
         parquet_read_options=ParquetReadOptions(coerce_int96_timestamp_unit="ms"),
         filesystem=auth_client.get_pyarrow_filesystem(path),
@@ -90,7 +106,7 @@ def load(  # pylint: disable=R0913
     return pyarrow_table.to_pandas(timestamp_as_object=True)
 
 
-def history(auth_client: AuthenticationClient, path: DataPath, limit: Optional[int] = 1) -> Iterable[DeltaTransaction]:
+def history(auth_client: AuthenticationClient, path: DataPath, limit: int | None = 1) -> Iterable[DeltaTransaction]:
     """
       Returns transaction history for the table under path.
 
@@ -108,10 +124,10 @@ def get_cache_key(
     auth_client: AuthenticationClient,
     path: DataPath,
     batch_size=1000,
-    version: Optional[int] = None,
-    row_filter: Optional[Expression] = None,
-    columns: Optional[List[str]] = None,
-    partition_filter_expressions: Optional[List[Tuple]] = None,
+    version: int | None = None,
+    row_filter: Expression | None = None,
+    columns: list[str] | None = None,
+    partition_filter_expressions: list[tuple] | None = None,
 ) -> str:
     """
       Returns a cache key for the path and data read arguments
@@ -156,13 +172,13 @@ def load_cached(  # pylint: disable=R0913
     auth_client: AuthenticationClient,
     path: DataPath,
     cache: KeyValueCache,
-    cache_expires_after: Optional[datetime.timedelta] = datetime.timedelta(hours=1),
+    cache_expires_after: datetime.timedelta | None = datetime.timedelta(hours=1),
     batch_size=1000,
-    version: Optional[int] = None,
-    row_filter: Optional[Expression] = None,
-    columns: Optional[List[str]] = None,
-    partition_filter_expressions: Optional[List[Tuple]] = None,
-    logger: Optional[SemanticLogger] = None,
+    version: int | None = None,
+    row_filter: Expression | None = None,
+    columns: list[str] | None = None,
+    partition_filter_expressions: list[tuple] | None = None,
+    logger: SemanticLogger | None = None,
 ) -> DataFrame:
     """
      Loads Delta Lake table from an external cache and converts it to a single pandas dataframe (after applying column projections and row filters).
@@ -217,7 +233,7 @@ def load_cached(  # pylint: disable=R0913
         try:
             return concat(
                 [
-                    DataFrameParquetSerializationFormat().deserialize(zlib.decompress(cached_batch))
+                    PandasDataFrameParquetSerializationFormat().deserialize(zlib.decompress(cached_batch))
                     for batch_key, cached_batch in cache.get(cache_key, is_map=True).items()
                     if batch_key != b"completed"
                 ]
@@ -255,7 +271,7 @@ def load_cached(  # pylint: disable=R0913
             cache.include(
                 key=cache_key,
                 attribute=str(batch_index),
-                value=zlib.compress(DataFrameParquetSerializationFormat().serialize(batch)),
+                value=zlib.compress(PandasDataFrameParquetSerializationFormat().serialize(batch)),
             )
             for batch_index, batch in enumerate(data)
         ],

adapta/storage/delta_lake/v2/_models.py

@@ -0,0 +1,72 @@
+# pylint: disable=duplicate-code
+"""
+ Models used by delta lake functions.
+"""
+#  Copyright (c) 2023-2026. ECCO Data & AI and other project contributors.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class DeltaOperation(Enum):
+    """
+    Possible Delta table operations.
+    """
+
+    DELETE = "DELETE"
+    UPDATE = "UPDATE"
+    WRITE = "WRITE"
+    MERGE = "MERGE"
+    CREATE_TABLE = "CREATE TABLE"
+    CREATE_TABLE_AS_SELECT = "CREATE TABLE AS SELECT"
+    CREATE_OR_REPLACE_TABLE_AS_SELECT = "CREATE OR REPLACE TABLE AS SELECT"
+    CHANGE_COLUMN = "CHANGE COLUMN"
+    VACUUM_START = "VACUUM START"
+    VACUUM_END = "VACUUM END"
+    UNDEFINED = "UNDEFINED"
+
+
+@dataclass
+class DeltaTransaction:
+    """
+    A subset of Delta table transaction entry properties.
+    """
+
+    version: int
+    timestamp: int
+    operation: DeltaOperation
+    operation_parameters: dict
+    read_version: int
+    is_blind_append: bool
+
+    @classmethod
+    def from_dict(cls, value: dict) -> "DeltaTransaction":
+        """
+          Converts delta transaction log entry to DeltaTransaction.
+        :param value: single entry from `describe history ...`
+        :return:
+        """
+        delta_op = value.get("operation", DeltaOperation.UNDEFINED.value)
+        supported_ops = {item.value for item in DeltaOperation}
+
+        return cls(
+            version=value.get("version", -1),
+            timestamp=value["timestamp"],
+            operation=DeltaOperation(delta_op) if delta_op in supported_ops else DeltaOperation.UNDEFINED,
+            operation_parameters=value.get("operationParameters", {}),
+            read_version=value.get("readVersion", -1),
+            is_blind_append=value.get("isBlindAppend", False),
+        )

adapta/storage/delta_lake/v3/README.md

@@ -0,0 +1,147 @@
+# Delta Lake Operations
+
+Supported API:
+- read delta table as `MetaFrame` which can easily be converted to `pandas.DataFrame` or `polars.DataFrame`
+- read delta table in batches of a provided size, each batch being `MetaFrame`
+- read a subset of columns from delta table
+- read and filter a delta table without loading all rows in memory
+
+## Examples usage
+Prepare connection and load
+### For Azure Datalake Gen2
+
+```python
+import os
+from adapta.security.clients import AzureClient
+from adapta.storage.models.azure import AdlsGen2Path
+from adapta.storage.delta_lake import load
+
+os.environ["PROTEUS__USE_AZURE_CREDENTIAL"] = "1"
+azure_client = AzureClient()
+adls_path = AdlsGen2Path.from_hdfs_path('abfss://container@account.dfs.core.windows.net/path/to/my/table')
+
+# get Iterable[MetaFrame]
+batches = load(azure_client, adls_path, batch_size=1000)
+```
+
+### For AWS Simple Storage Service (S3) or S3-Compatible Storage
+
+```python
+import os
+from adapta.security.clients import AwsClient
+from adapta.security.clients.aws._aws_credentials import EnvironmentAwsCredentials
+from adapta.storage.delta_lake import load
+import pandas as pd
+import pyarrow as pa
+
+# Set up environment variables
+os.environ["PROTEUS__AWS_ACCESS_KEY_ID"] = minio_access_key_id
+os.environ["PROTEUS__AWS_SECRET_ACCESS_KEY"] = minio_secret_key
+os.environ["PROTEUS__AWS_REGION"] = "eu-central-1"
+os.environ["PROTEUS__AWS_ENDPOINT"] = "http://example.com"
+
+# Create client
+credentials = EnvironmentAwsCredentials()
+aws_client = AwsClient(credentials)
+
+# Initialize session
+aws_client.initialize_session()
+
+# Creating a delta lake table with sample data
+data = {
+    'Character': ['Boromir', 'Harry Potter', 'Sherlock Holmes', 'Tony Stark', 'Darth Vader'],
+    'Occupation': ['Professional succumber to temptation', 'Wizard', 'Detective', 'Iron Man', 'Sith Lord'],
+    'Catchphrase': [
+        'One does not simply walk into Mordor.',
+        'Expecto Patronum!',
+        'Elementary, my dear Watson.',
+        'I am Iron Man.',
+        'I find your lack of faith disturbing.'
+    ]
+}
+
+df = pd.DataFrame(data)  # Create a pandas DataFrame from the data
+table = pa.Table.from_pandas(df)  # Convert the DataFrame to a PyArrow Table
+path_test = '/path/to/store/locally/delta/lake/table'  
+deltalake.write_deltalake(path_test, table)  # Write the PyArrow Table to a Delta Lake table
+
+# Save the Delta Lake table to S3 blob storage
+s3_client.save_data(path_test, s3_path) 
+
+# Get Iterable[pandas.DataFrame]
+batches = load(aws_client, s3_path, batch_size=1000))
+
+# Print each loaded batch
+for batch in batches:
+    print(batch.to_pandas())
+    print("\n---\n")
+
+# The content of the Delta Lake table should be printed in the screen
+#         Character  ...                            Catchphrase
+# 0          Boromir  ...  One does not simply walk into Mordor.
+# 1     Harry Potter  ...                      Expecto Patronum!
+# 2  Sherlock Holmes  ...            Elementary, my dear Watson.
+# 3       Tony Stark  ...                         I am Iron Man.
+# 4      Darth Vader  ...  I find your lack of faith disturbing.
+# 
+# [5 rows x 3 columns]
+# ---
+```
+## Using the Filtering API.
+1. Create generic filter expressions
+```python
+from adapta.storage.models.filter_expression import FilterField
+
+simple_filter = FilterField("my_column") == "some-value"
+combined_filter = (FilterField("my_column") == "some-value") & (FilterField("other_column") == "another-value")
+combined_filter_with_collection = (FilterField("my_column") == "something1") & (FilterField("other_column").isin(['else', 'nonexistent']))
+complex_filter = (FilterField("my_column") == "something1") | (FilterField("my_other_column") == "else") & (FilterField("another_column") == 123)
+```
+2. Load and apply the expression
+```python
+# simple_filtered is of type pandas.DataFrame
+simple_filtered = load(azure_client, adls_path, row_filter=simple_expression_pyarrow, columns=["my_column", "my_other_column"]).to_pandas()
+#     my_column my_other_column
+# 0  some-value             123
+# 1  some-value   another-value
+
+print(load(azure_client, adls_path, row_filter=combined_filter, columns=["my_column", "my_other_column"]).to_pandas())
+#     my_column my_other_column
+# 0  some-value   another-value
+
+print(load(azure_client, adls_path, row_filter=combined_filter_with_collection, columns=["my_column", "my_other_column"]).to_pandas())
+#     my_column my_other_column
+# 0  something1            else
+# 1  something1     nonexistent
+
+print(load(azure_client, adls_path, row_filter=complex_filter, columns=["my_column", "my_other_column", "another_column"]).to_pandas())
+#     my_column my_other_column  another_column
+# 0  something1            else               1
+# 1  something1     nonexistent               2
+# 2  something1    nonexistent1             123
+
+```
+# Using with Hive paths
+```python
+logger: SemanticLogger  # review proteus.logs readme to learn how to construct a logger instance
+os.environ['PROTEUS__HIVE_USER'] = 'delamain'
+os.environ['PROTEUS__HIVE_PASSWORD'] = 'secret'
+hive_path = HivePath.from_hdfs_path(
+    "hive://sqlserver@myserver.database.windows.net:1433/sparkdatalake/bronze/bronze_table")
+
+adls_path2 = AdlsGen2Path.from_hdfs_path(hive_path.get_physical_path(logger=logger))
+
+# get Iterable[MetaFrame]
+batches2 = load(azure_client, adls_path2, batch_size=1000)
+
+# read data using Redis Cache, improves read time by a factor of >10 on single-node Redis.
+# for big tables, choose bigger batch sizes to speed up cache population. General rule:
+# batch_size = row_count / 10
+# if there is no cache hit, load_cached() will fallback to load() behaviour
+r_cache = RedisCache(host="esd-superset-test.redis.cache.windows.net", database_number=1)
+os.environ['PROTEUS__CACHE_REDIS_PASSWORD'] = '...'
+read_raw = load_cached(azure_client, adls_path, r_cache, row_filter=filter,
+                       cache_expires_after=datetime.timedelta(minutes=15), batch_size=int(1e6))
+```
+
+

adapta/storage/delta_lake/v3/__init__.py

@@ -0,0 +1,20 @@
+"""
+Import index
+"""
+#  Copyright (c) 2023-2026. ECCO Data & AI and other project contributors.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from adapta.storage.delta_lake.v3._functions import *
+from adapta.storage.delta_lake.v3._models import *