deltacat 2.0.0b11__py3-none-any.whl → 2.0.0.post1__py3-none-any.whl

deltacat/tests/test_exceptions.py

@@ -11,15 +11,15 @@ from deltacat.exceptions import (
     UnclassifiedDeltaCatError,
 )
 from daft.exceptions import DaftTransientError
-from deltacat.tests.local_deltacat_storage.exceptions import (
+from deltacat.tests.utils.exceptions import (
     InvalidNamespaceError,
-    LocalStorageValidationError,
+    MainStorageValidationError,
 )
+from deltacat.tests.utils import main_deltacat_storage_mock as ds
 from botocore.exceptions import NoCredentialsError
 from tenacity import retry, retry_if_exception_type, stop_after_attempt
 
 from pyarrow.lib import ArrowCapacityError
-import deltacat.tests.local_deltacat_storage as ds
 
 
 class MockUnknownException(Exception):
@@ -41,7 +41,7 @@ def mock_remote_task(exception_to_raise):
     mock_raise_exception(exception_to_raise)
 
 
-class TestCategorizeErrors(unittest.TestCase):
+class TestCategorizeErrorsMain(unittest.TestCase):
     def test_pyarrow_exception_categorizer(self):
         self.assertRaises(
             DependencyPyarrowCapacityError,
@@ -50,7 +50,7 @@ class TestCategorizeErrors(unittest.TestCase):
 
     def test_storage_exception_categorizer(self):
         self.assertRaises(
-            LocalStorageValidationError,
+            MainStorageValidationError,
             lambda: mock_raise_exception(InvalidNamespaceError, deltacat_storage=ds),
         )
 
@@ -98,3 +98,7 @@ class TestCategorizeErrors(unittest.TestCase):
             return
 
         self.assertFalse(True)
+
+
+if __name__ == "__main__":
+    unittest.main()

deltacat/tests/test_utils/pyarrow.py

@@ -1,8 +1,9 @@
 from typing import List, Optional, Union
 import pyarrow as pa
 from deltacat.storage import Delta, Partition, PartitionLocator, DeltaLocator
-import deltacat.tests.local_deltacat_storage as ds
+from deltacat.storage import metastore
 from deltacat.types.media import StorageType, ContentType
+from deltacat.storage.model.schema import Schema
 
 
 def create_delta_from_csv_file(
@@ -14,58 +15,89 @@ def create_delta_from_csv_file(
     *args,
     **kwargs,
 ) -> Delta:
+    assert file_paths is not None, "file_paths cannot be empty"
+    pa_table = create_table_from_csv_file_paths(file_paths)
+    schema = Schema.of(pa_table.schema)
     staged_partition = stage_partition_from_file_paths(
         namespace,
         file_paths,
+        schema,
         *args,
         table_name=table_name,
         table_version=table_version,
         **kwargs,
     )
     committed_delta = commit_delta_to_staged_partition(
-        staged_partition, file_paths, content_type=content_type, *args, **kwargs
+        staged_partition,
+        pa_table,
+        content_type,
+        *args,
+        **kwargs,
     )
     return committed_delta
 
 
+def create_table_from_csv_file_paths(
+    file_paths: List[str],
+) -> pa.Table:
+    tables = []
+    for file_path in file_paths:
+        table = pa.csv.read_csv(file_path)
+        tables.append(table)
+    return pa.concat_tables(tables)
+
+
 def stage_partition_from_file_paths(
     namespace: str,
     file_paths: List[str],
+    schema: Schema,
     table_name: Optional[str] = None,
     table_version: int = 1,
     *args,
     **kwargs,
 ) -> Partition:
-    ds.create_namespace(namespace, {}, **kwargs)
+    if not metastore.namespace_exists(namespace, **kwargs):
+        metastore.create_namespace(namespace, **kwargs)
     if table_name is None:
         table_name = "-".join(file_paths).replace("/", "_")
-    ds.create_table_version(namespace, table_name, str(table_version), **kwargs)
-    stream = ds.get_stream(namespace, table_name, str(table_version), **kwargs)
-    staged_partition = ds.stage_partition(stream, [], **kwargs)
+    metastore.create_table_version(
+        namespace,
+        table_name,
+        str(table_version),
+        schema=schema,
+        **kwargs,
+    )
+    stream = metastore.get_stream(
+        namespace,
+        table_name,
+        str(table_version),
+        **kwargs,
+    )
+    staged_partition = metastore.stage_partition(stream, **kwargs)
     return staged_partition
 
 
 def commit_delta_to_staged_partition(
     staged_partition,
-    file_paths: List[str],
+    pa_table: pa.Table,
     content_type: ContentType = ContentType.PARQUET,
     *args,
     **kwargs,
 ) -> Delta:
     committed_delta = commit_delta_to_partition(
         staged_partition,
+        pa_table,
+        content_type,
         *args,
-        file_paths=file_paths,
-        content_type=content_type,
         **kwargs,
     )
-    ds.commit_partition(staged_partition, **kwargs)
+    metastore.commit_partition(staged_partition, **kwargs)
     return committed_delta
 
 
 def download_delta(delta_like: Union[Delta, DeltaLocator], *args, **kwargs) -> Delta:
     return pa.concat_tables(
-        ds.download_delta(
+        metastore.download_delta(
             delta_like,
             storage_type=StorageType.LOCAL,
             *args,
@@ -76,23 +108,22 @@ def download_delta(delta_like: Union[Delta, DeltaLocator], *args, **kwargs) -> D
 
 def commit_delta_to_partition(
     partition: Union[Partition, PartitionLocator],
-    file_paths: List[str],
+    pa_table: pa.Table = None,
     content_type: ContentType = ContentType.PARQUET,
     *args,
     **kwargs,
 ) -> Delta:
-    tables = []
 
     if isinstance(partition, PartitionLocator):
-        partition = ds.get_partition(
+        partition = metastore.get_partition(
             partition.stream_locator, partition.partition_values, *args, **kwargs
         )
 
-    for file_path in file_paths:
-        table = pa.csv.read_csv(file_path)
-        tables.append(table)
-
-    table = pa.concat_tables(tables)
-    staged_delta = ds.stage_delta(table, partition, content_type=content_type, **kwargs)
+    staged_delta = metastore.stage_delta(
+        pa_table,
+        partition,
+        content_type=content_type,
+        **kwargs,
+    )
 
-    return ds.commit_delta(staged_delta, **kwargs)
+    return metastore.commit_delta(staged_delta, **kwargs)

deltacat/tests/test_utils/storage.py

@@ -25,12 +25,16 @@ from deltacat.storage import (
     NullOrder,
     Partition,
     PartitionKey,
+    PartitionKeyList,
     PartitionLocator,
     PartitionScheme,
+    PartitionSchemeList,
     Schema,
     SchemaList,
     SortScheme,
+    SortSchemeList,
     SortKey,
+    SortKeyList,
     SortOrder,
     StreamLocator,
     StreamFormat,
@@ -59,7 +63,10 @@ def create_empty_delta(
     manifest_entry_id: Optional[str] = None,
 ) -> Delta:
     stream_position = current_time_ms()
-    delta_locator = DeltaLocator.of(partition.locator, stream_position=stream_position)
+    delta_locator = DeltaLocator.of(
+        partition.locator,
+        stream_position=stream_position,
+    )
 
     if manifest_entry_id:
         manifest = Manifest.of(
@@ -131,12 +138,12 @@ def create_test_table_version():
         PartitionKey.of(
             key=["some_string", "some_int32"],
             name="test_partition_key",
-            field_id="test_field_id",
+            field_id=1,
             transform=bucket_transform,
         )
     ]
     partition_scheme = PartitionScheme.of(
-        keys=partition_keys,
+        keys=PartitionKeyList.of(partition_keys),
         name="test_partition_scheme",
         scheme_id="test_partition_scheme_id",
     )
@@ -151,7 +158,7 @@ def create_test_table_version():
         )
     ]
     sort_scheme = SortScheme.of(
-        keys=sort_keys,
+        keys=SortKeyList.of(sort_keys),
         name="test_sort_scheme",
         scheme_id="test_sort_scheme_id",
     )
@@ -166,8 +173,8 @@ def create_test_table_version():
         watermark=None,
         lifecycle_state=LifecycleState.CREATED,
         schemas=SchemaList.of([schema]),
-        partition_schemes=[partition_scheme],
-        sort_schemes=[sort_scheme],
+        partition_schemes=PartitionSchemeList.of([partition_scheme]),
+        sort_schemes=SortSchemeList.of([sort_scheme]),
     )
 
 
@@ -189,12 +196,12 @@ def create_test_stream():
         PartitionKey.of(
             key=["some_string", "some_int32"],
             name="test_partition_key",
-            field_id="test_field_id",
+            field_id=1,
             transform=bucket_transform,
         )
     ]
     partition_scheme = PartitionScheme.of(
-        keys=partition_keys,
+        keys=PartitionKeyList.of(partition_keys),
         name="test_partition_scheme",
         scheme_id="test_partition_scheme_id",
     )
@@ -217,28 +224,8 @@ def create_test_partition():
         partition_values=["a", 1],
         partition_id="test_partition_id",
     )
-    schema = Schema.of(
-        [
-            Field.of(
-                field=pa.field("some_string", pa.string(), nullable=False),
-                field_id=1,
-                is_merge_key=True,
-            ),
-            Field.of(
-                field=pa.field("some_int32", pa.int32(), nullable=False),
-                field_id=2,
-                is_merge_key=True,
-            ),
-            Field.of(
-                field=pa.field("some_float64", pa.float64()),
-                field_id=3,
-                is_merge_key=False,
-            ),
-        ]
-    )
     return Partition.of(
         locator=partition_locator,
-        schema=schema,
         content_types=[ContentType.PARQUET],
         state=CommitState.STAGED,
         previous_stream_position=0,
@@ -274,12 +261,14 @@ def create_test_delta():
         entry_params=manifest_entry_params,
     )
     manifest = Manifest.of(
-        entries=[
-            ManifestEntry.of(
-                url="s3://test/url",
-                meta=manifest_meta,
-            )
-        ],
+        entries=ManifestEntryList(
+            [
+                ManifestEntry.of(
+                    url="s3://test/url",
+                    meta=manifest_meta,
+                )
+            ]
+        ),
         author=ManifestAuthor.of(
             name="deltacat",
             version="2.0",

deltacat/tests/types/test_tables.py

@@ -0,0 +1,104 @@
+import pytest
+import pandas as pd
+import pyarrow as pa
+
+from deltacat.types.tables import (
+    to_pandas,
+    to_pyarrow,
+    get_table_length,
+)
+
+
+def test_convert_to_pandas_error_cases():
+    """Test convert_to_pandas with invalid inputs."""
+    # Test None input
+    with pytest.raises(
+        ValueError, match="No pandas conversion function found for table type"
+    ):
+        to_pandas(None)
+
+    # Test unsupported type
+    with pytest.raises(
+        ValueError, match="No pandas conversion function found for table type"
+    ):
+        to_pandas("invalid_string")
+
+    # Test unsupported type with complex object
+    with pytest.raises(
+        ValueError, match="No pandas conversion function found for table type"
+    ):
+        to_pandas({"not": "a_dataframe"})
+
+
+def test_convert_to_arrow_error_cases():
+    """Test convert_to_arrow with invalid inputs."""
+    # Test None input
+    with pytest.raises(
+        ValueError, match="No pyarrow conversion function found for table type"
+    ):
+        to_pyarrow(None)
+
+    # Test unsupported type
+    with pytest.raises(
+        ValueError, match="No pyarrow conversion function found for table type"
+    ):
+        to_pyarrow("invalid_string")
+
+    # Test unsupported type with complex object
+    with pytest.raises(
+        ValueError, match="No pyarrow conversion function found for table type"
+    ):
+        to_pyarrow({"not": "a_table"})
+
+
+def test_conversion_functions_with_real_data():
+    """Test conversion functions with actual data structures."""
+    # Create test data
+    test_df = pd.DataFrame({"id": [1, 2], "name": ["test1", "test2"]})
+    test_table = pa.Table.from_pandas(test_df)
+
+    # Test pandas conversion
+    converted_df = to_pandas(test_df)
+    assert isinstance(converted_df, pd.DataFrame)
+    assert converted_df.equals(test_df)
+
+    # Test arrow conversion
+    converted_table = to_pyarrow(test_table)
+    assert isinstance(converted_table, pa.Table)
+    assert converted_table.equals(test_table)
+
+    # Test cross-conversion
+    df_from_table = to_pandas(test_table)
+    table_from_df = to_pyarrow(test_df)
+    assert isinstance(df_from_table, pd.DataFrame)
+    assert isinstance(table_from_df, pa.Table)
+
+
+def test_conversion_roundtrip_consistency():
+    """Test that conversion functions maintain data integrity through roundtrips."""
+    # Create test data
+    original_df = pd.DataFrame(
+        {
+            "id": [1, 2, 3, 4, 5],
+            "name": ["Alice", "Bob", "Charlie", "Dave", "Eve"],
+            "age": [25, 30, 35, 40, 45],
+            "city": ["NYC", "LA", "Chicago", "Houston", "Phoenix"],
+        }
+    )
+
+    # Test pandas -> arrow -> pandas roundtrip
+    arrow_table = to_pyarrow(original_df)
+    roundtrip_df = to_pandas(arrow_table)
+
+    # Verify data integrity (allowing for potential type changes)
+    assert get_table_length(original_df) == get_table_length(
+        roundtrip_df
+    ), "Row count should be preserved"
+    assert list(original_df.columns) == list(
+        roundtrip_df.columns
+    ), "Column names should be preserved"
+
+    # Verify ID column integrity (critical for merge operations)
+    original_ids = sorted(original_df["id"].tolist())
+    roundtrip_ids = sorted(roundtrip_df["id"].tolist())
+    assert original_ids == roundtrip_ids, "ID column should be preserved exactly"

deltacat/tests/utils/exceptions.py

@@ -0,0 +1,22 @@
+"""
+Exception classes for main storage testing that mirror the local storage exceptions.
+These are used to test the main metastore error categorization functionality.
+"""
+
+
+class InvalidNamespaceError(Exception):
+    """Exception raised when an invalid namespace is provided to main storage."""
+
+    error_name = "InvalidNamespaceError"
+
+
+class MainStorageValidationError(Exception):
+    """Exception raised when main storage validation fails."""
+
+    error_name = "MainStorageValidationError"
+
+
+class MainStorageError(Exception):
+    """General exception for main storage operations."""
+
+    error_name = "MainStorageError"

deltacat/tests/utils/main_deltacat_storage_mock.py

@@ -0,0 +1,31 @@
+"""
+Mock module that provides storage-specific error categorization functions for main storage testing.
+"""
+
+from deltacat.tests.utils.exceptions import (
+    InvalidNamespaceError,
+    MainStorageValidationError,
+)
+
+
+def can_categorize(e: BaseException, **kwargs) -> bool:
+    """
+    Mock implementation of can_categorize for main storage testing.
+    Returns True if the input error can be categorized by main storage.
+    """
+    if isinstance(e, InvalidNamespaceError):
+        return True
+    else:
+        return False
+
+
+def raise_categorized_error(e: BaseException, **kwargs):
+    """
+    Mock implementation of raise_categorized_error for main storage testing.
+    Converts categorizable errors to their main storage equivalent.
+    """
+    if isinstance(e, InvalidNamespaceError):
+        raise MainStorageValidationError("Namespace provided is invalid!")
+    else:
+        # If we can't categorize it, re-raise the original exception
+        raise e

deltacat/tests/utils/ray_utils/test_dataset.py

@@ -6,6 +6,8 @@ from fsspec import AbstractFileSystem
 from ray.data.datasource import FilenameProvider
 from deltacat.types.media import ContentType
 import ray
+import gzip
+import json
 
 
 class TestDatasetToFile:
@@ -20,7 +22,13 @@ class TestDatasetToFile:
 
     @pytest.fixture(scope="module")
     def mock_dataset(self):
-        return from_items([{"col1": i, "col2": i * 2} for i in range(1000)])
+        # Include data that would need escaping to test quoting behavior
+        return from_items([{"col1": "a,b\tc|d", "col2": 0} for _ in range(5)])
+
+    @pytest.fixture(scope="module")
+    def mock_unescaped_dataset(self):
+        # Use data without delimiters for unescaped TSV test
+        return from_items([{"col1": "abc", "col2": 0} for _ in range(5)])
 
     @pytest.fixture(scope="module")
     def mock_filename_provider(self):
@@ -35,12 +43,12 @@ class TestDatasetToFile:
     def test_parquet_sanity(self, mock_dataset, mock_filename_provider):
         from deltacat.utils.ray_utils.dataset import dataset_to_file
 
-        fs: AbstractFileSystem = fsspec.filesystem("local")
+        fs: AbstractFileSystem = fsspec.filesystem("file")
 
         dataset_to_file(
             mock_dataset,
             self.BASE_PATH,
-            file_system=fs,
+            filesystem=fs,
             block_path_provider=mock_filename_provider,
         )
 
@@ -51,16 +59,126 @@ class TestDatasetToFile:
     def test_csv_sanity(self, mock_dataset, mock_filename_provider):
         from deltacat.utils.ray_utils.dataset import dataset_to_file
 
-        fs: AbstractFileSystem = fsspec.filesystem("local")
+        fs: AbstractFileSystem = fsspec.filesystem("file")
 
         dataset_to_file(
             mock_dataset,
             self.BASE_PATH,
-            file_system=fs,
+            filesystem=fs,
             block_path_provider=mock_filename_provider,
             content_type=ContentType.CSV.value,
         )
 
         file_expected_at = f"{self.BASE_PATH}/{self.SUB_PATH}"
         assert fs.exists(file_expected_at), "file was not written"
+
+        # Verify CSV format and content
+        with fs.open(file_expected_at, "rb") as f:
+            with gzip.GzipFile(fileobj=f) as gz:
+                content = gz.read().decode("utf-8")
+                # Should be quoted due to commas in data
+                assert '"a,b\tc|d",0' in content
+
+        fs.delete(file_expected_at)
+
+    def test_tsv_sanity(self, mock_dataset, mock_filename_provider):
+        from deltacat.utils.ray_utils.dataset import dataset_to_file
+
+        fs: AbstractFileSystem = fsspec.filesystem("file")
+
+        dataset_to_file(
+            mock_dataset,
+            self.BASE_PATH,
+            filesystem=fs,
+            block_path_provider=mock_filename_provider,
+            content_type=ContentType.TSV.value,
+        )
+
+        file_expected_at = f"{self.BASE_PATH}/{self.SUB_PATH}"
+        assert fs.exists(file_expected_at), "file was not written"
+
+        # Verify TSV format and content
+        with fs.open(file_expected_at, "rb") as f:
+            with gzip.GzipFile(fileobj=f) as gz:
+                content = gz.read().decode("utf-8")
+                # Should be quoted due to tabs in data
+                assert '"a,b\tc|d"\t0' in content
+
+        fs.delete(file_expected_at)
+
+    def test_psv_sanity(self, mock_dataset, mock_filename_provider):
+        from deltacat.utils.ray_utils.dataset import dataset_to_file
+
+        fs: AbstractFileSystem = fsspec.filesystem("file")
+
+        dataset_to_file(
+            mock_dataset,
+            self.BASE_PATH,
+            filesystem=fs,
+            block_path_provider=mock_filename_provider,
+            content_type=ContentType.PSV.value,
+        )
+
+        file_expected_at = f"{self.BASE_PATH}/{self.SUB_PATH}"
+        assert fs.exists(file_expected_at), "file was not written"
+
+        # Verify PSV format and content
+        with fs.open(file_expected_at, "rb") as f:
+            with gzip.GzipFile(fileobj=f) as gz:
+                content = gz.read().decode("utf-8")
+                # Should be quoted due to pipes in data
+                assert '"a,b\tc|d"|0' in content
+
+        fs.delete(file_expected_at)
+
+    def test_unescaped_tsv_sanity(self, mock_unescaped_dataset, mock_filename_provider):
+        from deltacat.utils.ray_utils.dataset import dataset_to_file
+
+        fs: AbstractFileSystem = fsspec.filesystem("file")
+
+        dataset_to_file(
+            mock_unescaped_dataset,
+            self.BASE_PATH,
+            filesystem=fs,
+            block_path_provider=mock_filename_provider,
+            content_type=ContentType.UNESCAPED_TSV.value,
+        )
+
+        file_expected_at = f"{self.BASE_PATH}/{self.SUB_PATH}"
+        assert fs.exists(file_expected_at), "file was not written"
+
+        # Verify UNESCAPED_TSV format and content
+        with fs.open(file_expected_at, "rb") as f:
+            with gzip.GzipFile(fileobj=f) as gz:
+                content = gz.read().decode("utf-8")
+                # Should NOT be quoted since data has no delimiters
+                assert "abc\t0" in content
+
+        fs.delete(file_expected_at)
+
+    def test_json_sanity(self, mock_dataset, mock_filename_provider):
+        from deltacat.utils.ray_utils.dataset import dataset_to_file
+
+        fs: AbstractFileSystem = fsspec.filesystem("file")
+
+        dataset_to_file(
+            mock_dataset,
+            self.BASE_PATH,
+            filesystem=fs,
+            block_path_provider=mock_filename_provider,
+            content_type=ContentType.JSON.value,
+        )
+
+        file_expected_at = f"{self.BASE_PATH}/{self.SUB_PATH}"
+        assert fs.exists(file_expected_at), "file was not written"
+
+        # Verify JSON format and content
+        with fs.open(file_expected_at, "rb") as f:
+            with gzip.GzipFile(fileobj=f) as gz:
+                content = gz.read().decode("utf-8")
+                # Each line should be a valid JSON object
+                first_line = content.split("\n")[0]
+                record = json.loads(first_line)
+                assert record == {"col1": "a,b\tc|d", "col2": 0}
+
         fs.delete(file_expected_at)