deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/tests/test_exceptions.py

@@ -11,15 +11,15 @@ from deltacat.exceptions import (
     UnclassifiedDeltaCatError,
 )
 from daft.exceptions import DaftTransientError
-from deltacat.tests.local_deltacat_storage.exceptions import (
+from deltacat.tests.utils.exceptions import (
     InvalidNamespaceError,
-    LocalStorageValidationError,
+    MainStorageValidationError,
 )
+from deltacat.tests.utils import main_deltacat_storage_mock as ds
 from botocore.exceptions import NoCredentialsError
 from tenacity import retry, retry_if_exception_type, stop_after_attempt
 
 from pyarrow.lib import ArrowCapacityError
-import deltacat.tests.local_deltacat_storage as ds
 
 
 class MockUnknownException(Exception):
@@ -41,7 +41,7 @@ def mock_remote_task(exception_to_raise):
     mock_raise_exception(exception_to_raise)
 
 
-class TestCategorizeErrors(unittest.TestCase):
+class TestCategorizeErrorsMain(unittest.TestCase):
     def test_pyarrow_exception_categorizer(self):
         self.assertRaises(
             DependencyPyarrowCapacityError,
@@ -50,7 +50,7 @@ class TestCategorizeErrors(unittest.TestCase):
 
     def test_storage_exception_categorizer(self):
         self.assertRaises(
-            LocalStorageValidationError,
+            MainStorageValidationError,
             lambda: mock_raise_exception(InvalidNamespaceError, deltacat_storage=ds),
         )
 
@@ -98,3 +98,7 @@ class TestCategorizeErrors(unittest.TestCase):
             return
 
         self.assertFalse(True)
+
+
+if __name__ == "__main__":
+    unittest.main()

deltacat/tests/test_utils/filesystem.py

@@ -0,0 +1,14 @@
+import shutil
+import tempfile
+
+
+from contextlib import contextmanager
+
+
+@contextmanager
+def temp_dir_autocleanup():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        yield tmpdir
+    finally:
+        shutil.rmtree(tmpdir)

deltacat/tests/test_utils/message_pack_utils.py

@@ -0,0 +1,54 @@
+import base64
+import msgpack
+import json
+import os
+import shutil
+
+from tempfile import mkdtemp
+
+
+def _convert_bytes_to_base64_str(obj):
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if isinstance(value, bytes):
+                obj[key] = base64.b64encode(value).decode("utf-8")
+            elif isinstance(value, list):
+                _convert_bytes_to_base64_str(value)
+            elif isinstance(value, dict):
+                _convert_bytes_to_base64_str(value)
+    elif isinstance(obj, list):
+        for i, item in enumerate(obj):
+            if isinstance(item, bytes):
+                obj[i] = base64.b64encode(item).decode("utf-8")
+            elif isinstance(item, (dict, list)):
+                _convert_bytes_to_base64_str(item)
+
+
+def copy_and_convert(src_dir, dst_dir=None):
+    """
+    Helper function for copying a metastore recursively and converting all
+    messagepack files to json. This can be used manually to more easily
+    introspect metastore metadata.
+    """
+    if dst_dir is None:
+        dst_dir = mkdtemp()
+        print(f"destination is: {dst_dir}")
+    if not os.path.exists(dst_dir):
+        os.makedirs(dst_dir)
+
+    for item in os.listdir(src_dir):
+        src_path = os.path.join(src_dir, item)
+        dst_path = os.path.join(dst_dir, item)
+
+        if os.path.isdir(src_path):
+            copy_and_convert(src_path, dst_path)
+        else:
+            if item.endswith(".mpk"):
+                with open(src_path, "rb") as f:
+                    data = msgpack.unpackb(f.read(), raw=False)
+                    _convert_bytes_to_base64_str(data)
+                dst_path = dst_path[:-4] + ".json"
+                with open(dst_path, "w") as f:
+                    json.dump(data, f)
+            else:
+                shutil.copy2(src_path, dst_path)

deltacat/tests/test_utils/pyarrow.py

@@ -1,8 +1,9 @@
 from typing import List, Optional, Union
 import pyarrow as pa
 from deltacat.storage import Delta, Partition, PartitionLocator, DeltaLocator
-import deltacat.tests.local_deltacat_storage as ds
+from deltacat.storage import metastore
 from deltacat.types.media import StorageType, ContentType
+from deltacat.storage.model.schema import Schema
 
 
 def create_delta_from_csv_file(
@@ -14,60 +15,89 @@ def create_delta_from_csv_file(
     *args,
     **kwargs,
 ) -> Delta:
+    assert file_paths is not None, "file_paths cannot be empty"
+    pa_table = create_table_from_csv_file_paths(file_paths)
+    schema = Schema.of(pa_table.schema)
     staged_partition = stage_partition_from_file_paths(
         namespace,
         file_paths,
+        schema,
         *args,
         table_name=table_name,
         table_version=table_version,
         **kwargs,
     )
     committed_delta = commit_delta_to_staged_partition(
-        staged_partition, file_paths, content_type=content_type, *args, **kwargs
+        staged_partition,
+        pa_table,
+        content_type,
+        *args,
+        **kwargs,
     )
     return committed_delta
 
 
+def create_table_from_csv_file_paths(
+    file_paths: List[str],
+) -> pa.Table:
+    tables = []
+    for file_path in file_paths:
+        table = pa.csv.read_csv(file_path)
+        tables.append(table)
+    return pa.concat_tables(tables)
+
+
 def stage_partition_from_file_paths(
     namespace: str,
     file_paths: List[str],
+    schema: Schema,
     table_name: Optional[str] = None,
     table_version: int = 1,
     *args,
     **kwargs,
 ) -> Partition:
-    ds.create_namespace(namespace, {}, **kwargs)
+    if not metastore.namespace_exists(namespace, **kwargs):
+        metastore.create_namespace(namespace, **kwargs)
     if table_name is None:
         table_name = "-".join(file_paths).replace("/", "_")
-    ds.create_table_version(namespace, table_name, str(table_version), **kwargs)
-    stream = ds.get_stream(namespace, table_name, str(table_version), **kwargs)
-    staged_partition = ds.stage_partition(stream, [], **kwargs)
+    metastore.create_table_version(
+        namespace,
+        table_name,
+        str(table_version),
+        schema=schema,
+        **kwargs,
+    )
+    stream = metastore.get_stream(
+        namespace,
+        table_name,
+        str(table_version),
+        **kwargs,
+    )
+    staged_partition = metastore.stage_partition(stream, **kwargs)
     return staged_partition
 
 
 def commit_delta_to_staged_partition(
     staged_partition,
-    file_paths: List[str] = None,
-    pa_table: pa.Table = None,
+    pa_table: pa.Table,
     content_type: ContentType = ContentType.PARQUET,
     *args,
     **kwargs,
 ) -> Delta:
     committed_delta = commit_delta_to_partition(
         staged_partition,
+        pa_table,
+        content_type,
         *args,
-        file_paths=file_paths,
-        content_type=content_type,
-        pa_table=pa_table,
         **kwargs,
     )
-    ds.commit_partition(staged_partition, **kwargs)
+    metastore.commit_partition(staged_partition, **kwargs)
     return committed_delta
 
 
 def download_delta(delta_like: Union[Delta, DeltaLocator], *args, **kwargs) -> Delta:
     return pa.concat_tables(
-        ds.download_delta(
+        metastore.download_delta(
             delta_like,
             storage_type=StorageType.LOCAL,
             *args,
@@ -78,7 +108,6 @@ def download_delta(delta_like: Union[Delta, DeltaLocator], *args, **kwargs) -> D
 
 def commit_delta_to_partition(
     partition: Union[Partition, PartitionLocator],
-    file_paths: List[str] = None,
     pa_table: pa.Table = None,
     content_type: ContentType = ContentType.PARQUET,
     *args,
@@ -86,20 +115,15 @@ def commit_delta_to_partition(
 ) -> Delta:
 
     if isinstance(partition, PartitionLocator):
-        partition = ds.get_partition(
+        partition = metastore.get_partition(
             partition.stream_locator, partition.partition_values, *args, **kwargs
         )
-    if pa_table is None:
-        assert file_paths is not None, "One of pa_table or file_paths must be passed."
-        tables = []
-        for file_path in file_paths:
-            table = pa.csv.read_csv(file_path)
-            tables.append(table)
-
-        pa_table = pa.concat_tables(tables)
 
-    staged_delta = ds.stage_delta(
-        pa_table, partition, content_type=content_type, **kwargs
+    staged_delta = metastore.stage_delta(
+        pa_table,
+        partition,
+        content_type=content_type,
+        **kwargs,
     )
 
-    return ds.commit_delta(staged_delta, **kwargs)
+    return metastore.commit_delta(staged_delta, **kwargs)

deltacat/tests/test_utils/storage.py

@@ -1,7 +1,57 @@
 from typing import Optional, Dict
 
-from deltacat.aws.redshift import Manifest, ManifestMeta
-from deltacat.storage import Partition, DeltaType, DeltaLocator, Delta
+import pyarrow as pa
+
+from deltacat import (
+    ContentEncoding,
+    ContentType,
+)
+from deltacat.storage import (
+    BucketTransform,
+    BucketTransformParameters,
+    BucketingStrategy,
+    CommitState,
+    Delta,
+    DeltaLocator,
+    DeltaType,
+    EntryParams,
+    EntryType,
+    Field,
+    LifecycleState,
+    ManifestAuthor,
+    ManifestEntry,
+    Namespace,
+    NamespaceLocator,
+    NullOrder,
+    Partition,
+    PartitionKey,
+    PartitionKeyList,
+    PartitionLocator,
+    PartitionScheme,
+    PartitionSchemeList,
+    Schema,
+    SchemaList,
+    SortScheme,
+    SortSchemeList,
+    SortKey,
+    SortKeyList,
+    SortOrder,
+    StreamLocator,
+    StreamFormat,
+    Stream,
+    Table,
+    TableLocator,
+    TableVersionLocator,
+    TableVersion,
+    TruncateTransform,
+    TruncateTransformParameters,
+)
+
+from deltacat.storage.model.manifest import (
+    Manifest,
+    ManifestMeta,
+    ManifestEntryList,
+)
 from deltacat.utils.common import current_time_ms
 
 
@@ -13,11 +63,14 @@ def create_empty_delta(
     manifest_entry_id: Optional[str] = None,
 ) -> Delta:
     stream_position = current_time_ms()
-    delta_locator = DeltaLocator.of(partition.locator, stream_position=stream_position)
+    delta_locator = DeltaLocator.of(
+        partition.locator,
+        stream_position=stream_position,
+    )
 
     if manifest_entry_id:
         manifest = Manifest.of(
-            entries=[],
+            entries=ManifestEntryList.of([]),
             author=author,
             uuid=manifest_entry_id,
         )
@@ -32,3 +85,202 @@ def create_empty_delta(
         manifest=manifest,
         previous_stream_position=partition.stream_position,
     )
+
+
+def create_test_namespace():
+    namespace_locator = NamespaceLocator.of(namespace="test_namespace")
+    return Namespace.of(locator=namespace_locator)
+
+
+def create_test_table():
+    table_locator = TableLocator.at(
+        namespace="test_namespace",
+        table_name="test_table",
+    )
+    return Table.of(
+        locator=table_locator,
+        description="test table description",
+    )
+
+
+def create_test_table_version():
+    table_version_locator = TableVersionLocator.at(
+        namespace="test_namespace",
+        table_name="test_table",
+        table_version="v.1",
+    )
+    schema = Schema.of(
+        [
+            Field.of(
+                field=pa.field("some_string", pa.string(), nullable=False),
+                field_id=1,
+                is_merge_key=True,
+            ),
+            Field.of(
+                field=pa.field("some_int32", pa.int32(), nullable=False),
+                field_id=2,
+                is_merge_key=True,
+            ),
+            Field.of(
+                field=pa.field("some_float64", pa.float64()),
+                field_id=3,
+                is_merge_key=False,
+            ),
+        ]
+    )
+    bucket_transform = BucketTransform.of(
+        BucketTransformParameters.of(
+            num_buckets=2,
+            bucketing_strategy=BucketingStrategy.DEFAULT,
+        )
+    )
+    partition_keys = [
+        PartitionKey.of(
+            key=["some_string", "some_int32"],
+            name="test_partition_key",
+            field_id=1,
+            transform=bucket_transform,
+        )
+    ]
+    partition_scheme = PartitionScheme.of(
+        keys=PartitionKeyList.of(partition_keys),
+        name="test_partition_scheme",
+        scheme_id="test_partition_scheme_id",
+    )
+    sort_keys = [
+        SortKey.of(
+            key=["some_int32"],
+            sort_order=SortOrder.DESCENDING,
+            null_order=NullOrder.AT_START,
+            transform=TruncateTransform.of(
+                TruncateTransformParameters.of(width=3),
+            ),
+        )
+    ]
+    sort_scheme = SortScheme.of(
+        keys=SortKeyList.of(sort_keys),
+        name="test_sort_scheme",
+        scheme_id="test_sort_scheme_id",
+    )
+    return TableVersion.of(
+        locator=table_version_locator,
+        schema=schema,
+        partition_scheme=partition_scheme,
+        description="test table version description",
+        properties={"test_property_key": "test_property_value"},
+        content_types=[ContentType.PARQUET],
+        sort_scheme=sort_scheme,
+        watermark=None,
+        lifecycle_state=LifecycleState.CREATED,
+        schemas=SchemaList.of([schema]),
+        partition_schemes=PartitionSchemeList.of([partition_scheme]),
+        sort_schemes=SortSchemeList.of([sort_scheme]),
+    )
+
+
+def create_test_stream():
+    stream_locator = StreamLocator.at(
+        namespace="test_namespace",
+        table_name="test_table",
+        table_version="v.1",
+        stream_id="test_stream_id",
+        stream_format=StreamFormat.DELTACAT,
+    )
+    bucket_transform = BucketTransform.of(
+        BucketTransformParameters.of(
+            num_buckets=2,
+            bucketing_strategy=BucketingStrategy.DEFAULT,
+        )
+    )
+    partition_keys = [
+        PartitionKey.of(
+            key=["some_string", "some_int32"],
+            name="test_partition_key",
+            field_id=1,
+            transform=bucket_transform,
+        )
+    ]
+    partition_scheme = PartitionScheme.of(
+        keys=PartitionKeyList.of(partition_keys),
+        name="test_partition_scheme",
+        scheme_id="test_partition_scheme_id",
+    )
+    return Stream.of(
+        locator=stream_locator,
+        partition_scheme=partition_scheme,
+        state=CommitState.STAGED,
+        previous_stream_id="test_previous_stream_id",
+        watermark=1,
+    )
+
+
+def create_test_partition():
+    partition_locator = PartitionLocator.at(
+        namespace="test_namespace",
+        table_name="test_table",
+        table_version="v.1",
+        stream_id="test_stream_id",
+        stream_format=StreamFormat.DELTACAT,
+        partition_values=["a", 1],
+        partition_id="test_partition_id",
+    )
+    return Partition.of(
+        locator=partition_locator,
+        content_types=[ContentType.PARQUET],
+        state=CommitState.STAGED,
+        previous_stream_position=0,
+        previous_partition_id="test_previous_partition_id",
+        stream_position=1,
+        partition_scheme_id="test_partition_scheme_id",
+    )
+
+
+def create_test_delta():
+    delta_locator = DeltaLocator.at(
+        namespace="test_namespace",
+        table_name="test_table",
+        table_version="v.1",
+        stream_id="test_stream_id",
+        stream_format=StreamFormat.DELTACAT,
+        partition_values=["a", 1],
+        partition_id="test_partition_id",
+        stream_position=1,
+    )
+    manifest_entry_params = EntryParams.of(
+        equality_field_locators=["some_string", "some_int32"],
+    )
+    manifest_meta = ManifestMeta.of(
+        record_count=1,
+        content_length=10,
+        content_type=ContentType.PARQUET.value,
+        content_encoding=ContentEncoding.IDENTITY.value,
+        source_content_length=100,
+        credentials={"foo": "bar"},
+        content_type_parameters=[{"param1": "value1"}],
+        entry_type=EntryType.EQUALITY_DELETE,
+        entry_params=manifest_entry_params,
+    )
+    manifest = Manifest.of(
+        entries=ManifestEntryList(
+            [
+                ManifestEntry.of(
+                    url="s3://test/url",
+                    meta=manifest_meta,
+                )
+            ]
+        ),
+        author=ManifestAuthor.of(
+            name="deltacat",
+            version="2.0",
+        ),
+        entry_type=EntryType.EQUALITY_DELETE,
+        entry_params=manifest_entry_params,
+    )
+    return Delta.of(
+        locator=delta_locator,
+        delta_type=DeltaType.APPEND,
+        meta=manifest_meta,
+        properties={"property1": "value1"},
+        manifest=manifest,
+        previous_stream_position=0,
+    )

deltacat/tests/types/test_tables.py

@@ -0,0 +1,104 @@
+import pytest
+import pandas as pd
+import pyarrow as pa
+
+from deltacat.types.tables import (
+    to_pandas,
+    to_pyarrow,
+    get_table_length,
+)
+
+
+def test_convert_to_pandas_error_cases():
+    """Test convert_to_pandas with invalid inputs."""
+    # Test None input
+    with pytest.raises(
+        ValueError, match="No pandas conversion function found for table type"
+    ):
+        to_pandas(None)
+
+    # Test unsupported type
+    with pytest.raises(
+        ValueError, match="No pandas conversion function found for table type"
+    ):
+        to_pandas("invalid_string")
+
+    # Test unsupported type with complex object
+    with pytest.raises(
+        ValueError, match="No pandas conversion function found for table type"
+    ):
+        to_pandas({"not": "a_dataframe"})
+
+
+def test_convert_to_arrow_error_cases():
+    """Test convert_to_arrow with invalid inputs."""
+    # Test None input
+    with pytest.raises(
+        ValueError, match="No pyarrow conversion function found for table type"
+    ):
+        to_pyarrow(None)
+
+    # Test unsupported type
+    with pytest.raises(
+        ValueError, match="No pyarrow conversion function found for table type"
+    ):
+        to_pyarrow("invalid_string")
+
+    # Test unsupported type with complex object
+    with pytest.raises(
+        ValueError, match="No pyarrow conversion function found for table type"
+    ):
+        to_pyarrow({"not": "a_table"})
+
+
+def test_conversion_functions_with_real_data():
+    """Test conversion functions with actual data structures."""
+    # Create test data
+    test_df = pd.DataFrame({"id": [1, 2], "name": ["test1", "test2"]})
+    test_table = pa.Table.from_pandas(test_df)
+
+    # Test pandas conversion
+    converted_df = to_pandas(test_df)
+    assert isinstance(converted_df, pd.DataFrame)
+    assert converted_df.equals(test_df)
+
+    # Test arrow conversion
+    converted_table = to_pyarrow(test_table)
+    assert isinstance(converted_table, pa.Table)
+    assert converted_table.equals(test_table)
+
+    # Test cross-conversion
+    df_from_table = to_pandas(test_table)
+    table_from_df = to_pyarrow(test_df)
+    assert isinstance(df_from_table, pd.DataFrame)
+    assert isinstance(table_from_df, pa.Table)
+
+
+def test_conversion_roundtrip_consistency():
+    """Test that conversion functions maintain data integrity through roundtrips."""
+    # Create test data
+    original_df = pd.DataFrame(
+        {
+            "id": [1, 2, 3, 4, 5],
+            "name": ["Alice", "Bob", "Charlie", "Dave", "Eve"],
+            "age": [25, 30, 35, 40, 45],
+            "city": ["NYC", "LA", "Chicago", "Houston", "Phoenix"],
+        }
+    )
+
+    # Test pandas -> arrow -> pandas roundtrip
+    arrow_table = to_pyarrow(original_df)
+    roundtrip_df = to_pandas(arrow_table)
+
+    # Verify data integrity (allowing for potential type changes)
+    assert get_table_length(original_df) == get_table_length(
+        roundtrip_df
+    ), "Row count should be preserved"
+    assert list(original_df.columns) == list(
+        roundtrip_df.columns
+    ), "Column names should be preserved"
+
+    # Verify ID column integrity (critical for merge operations)
+    original_ids = sorted(original_df["id"].tolist())
+    roundtrip_ids = sorted(roundtrip_df["id"].tolist())
+    assert original_ids == roundtrip_ids, "ID column should be preserved exactly"

deltacat/tests/utils/exceptions.py

@@ -0,0 +1,22 @@
+"""
+Exception classes for main storage testing that mirror the local storage exceptions.
+These are used to test the main metastore error categorization functionality.
+"""
+
+
+class InvalidNamespaceError(Exception):
+    """Exception raised when an invalid namespace is provided to main storage."""
+
+    error_name = "InvalidNamespaceError"
+
+
+class MainStorageValidationError(Exception):
+    """Exception raised when main storage validation fails."""
+
+    error_name = "MainStorageValidationError"
+
+
+class MainStorageError(Exception):
+    """General exception for main storage operations."""
+
+    error_name = "MainStorageError"

deltacat/tests/utils/main_deltacat_storage_mock.py

@@ -0,0 +1,31 @@
+"""
+Mock module that provides storage-specific error categorization functions for main storage testing.
+"""
+
+from deltacat.tests.utils.exceptions import (
+    InvalidNamespaceError,
+    MainStorageValidationError,
+)
+
+
+def can_categorize(e: BaseException, **kwargs) -> bool:
+    """
+    Mock implementation of can_categorize for main storage testing.
+    Returns True if the input error can be categorized by main storage.
+    """
+    if isinstance(e, InvalidNamespaceError):
+        return True
+    else:
+        return False
+
+
+def raise_categorized_error(e: BaseException, **kwargs):
+    """
+    Mock implementation of raise_categorized_error for main storage testing.
+    Converts categorizable errors to their main storage equivalent.
+    """
+    if isinstance(e, InvalidNamespaceError):
+        raise MainStorageValidationError("Namespace provided is invalid!")
+    else:
+        # If we can't categorize it, re-raise the original exception
+        raise e