deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/tests/compute/test_util_constant.py

@@ -1,7 +1,6 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-TEST_S3_RCF_BUCKET_NAME = "test-compaction-artifacts-bucket"
 # REBASE  src = spark compacted table to create an initial version of ray compacted table
 BASE_TEST_SOURCE_NAMESPACE = "source_test_namespace"
 BASE_TEST_SOURCE_TABLE_NAME = "source_test_table"

deltacat/tests/conftest.py

@@ -0,0 +1,25 @@
+import tempfile
+
+import pytest
+from deltacat.catalog import CatalogProperties
+from deltacat.tests.test_utils.filesystem import temp_dir_autocleanup
+
+
+@pytest.fixture
+def temp_dir():
+    """
+    Temp dir which is removed after usage
+    note that each method which is injected with temp_dir will get a separate new tmp directory
+    """
+    with temp_dir_autocleanup() as tmp_dir:
+        yield tmp_dir
+
+
+@pytest.fixture
+def keep_temp_dir():
+    return tempfile.mkdtemp()
+
+
+@pytest.fixture
+def temp_catalog_properties(temp_dir):
+    return CatalogProperties(root=temp_dir)

deltacat/tests/daft/test_model.py

@@ -0,0 +1,97 @@
+import pytest
+import pyarrow as pa
+from daft import DataType, TimeUnit
+from daft.logical.schema import Field as DaftField
+
+from deltacat.storage.model.transform import IdentityTransform
+from deltacat.storage.model.partition import PartitionKey
+from deltacat.utils.daft import DaftFieldMapper, DaftPartitionKeyMapper
+
+from deltacat.storage.model.schema import Field, Schema
+
+
+class TestDaftFieldMapper:
+    def test_field_mapper_basic_types(self):
+        """Test mapping basic data types between Daft and PyArrow fields"""
+        test_cases = [
+            (DataType.int32(), pa.int32()),
+            (DataType.int64(), pa.int64()),
+            (DataType.float32(), pa.float32()),
+            (DataType.float64(), pa.float64()),
+            (DataType.string(), pa.large_string()),
+            (DataType.bool(), pa.bool_()),
+            (DataType.binary(), pa.large_binary()),
+            (DataType.date(), pa.date32()),
+            (DataType.timestamp(TimeUnit.ns()), pa.timestamp("ns")),
+        ]
+
+        for daft_type, pa_type in test_cases:
+            # Create test fields
+            daft_field = DaftField.create(
+                name="test_field",
+                dtype=daft_type,
+            )
+
+            # Daft to PyArrow
+            pa_field = DaftFieldMapper.map(daft_field)
+            assert pa_field is not None
+            assert pa_field.name == "test_field"
+            assert pa_field.type == pa_type  # type: ignore
+            assert pa_field.nullable is True
+
+            # PyArrow to Daft
+            daft_field_back = DaftFieldMapper.unmap(pa_field)
+            assert daft_field_back is not None
+            assert daft_field_back.name == daft_field.name
+            assert daft_field_back.dtype == daft_field.dtype
+
+
+class TestDaftPartitionKeyMapper:
+    def test_unmap(self):
+        """
+        Test unmap method of DaftPartitionKeyMapper when obj is not None, schema is provided,
+        len(obj.key) is 1, and dc_field is found in the schema.
+
+        This test verifies that the method correctly converts a PartitionKey to a DaftPartitionField
+        when all conditions are met and the field exists in the schema.
+        """
+        # Create a mock schema
+        schema = Schema.of(schema=[Field.of(pa.field("test_field", pa.int32()))])
+        # Create a PartitionKey object
+        partition_key = PartitionKey(
+            key=["test_field"], transform=IdentityTransform(), name="partition_field"
+        )
+
+        result = DaftPartitionKeyMapper.unmap(obj=partition_key, schema=schema)
+        assert result is not None
+        assert result.field.name() == "partition_field"
+        assert DataType._from_pydatatype(result.field.dtype()) == DataType.int32()
+
+    def test_unmap_no_field_locator(self):
+        schema = Schema.of(schema=[Field.of(pa.field("test_field", pa.int32()))])
+        partition_key = PartitionKey(key=[], name="partition_field")
+
+        with pytest.raises(ValueError) as excinfo:
+            DaftPartitionKeyMapper.unmap(partition_key, schema)
+
+        assert "At least 1 PartitionKey FieldLocator is expected" in str(excinfo.value)
+
+    def test_unmap_partition_key_not_found(self):
+        schema = Schema.of(schema=[Field.of(pa.field("test_field", pa.int32()))])
+        partition_key = PartitionKey(
+            key=["test_field_2"], transform=IdentityTransform(), name="partition_field"
+        )
+
+        with pytest.raises(KeyError) as excinfo:
+            DaftPartitionKeyMapper.unmap(partition_key, schema)
+
+        assert "Column test_field_2 does not exist in schema" in str(excinfo.value)
+
+    def test_unmap_partition_name_not_defined(self):
+        schema = Schema.of(schema=[Field.of(pa.field("test_field", pa.int32()))])
+        partition_key = PartitionKey(key=[])
+
+        with pytest.raises(ValueError) as excinfo:
+            DaftPartitionKeyMapper.unmap(partition_key, schema)
+
+        assert "Name is required for PartitionKey conversion" in str(excinfo.value)

deltacat/tests/experimental/__init__.py

@@ -0,0 +1 @@
+# Test package for experimental DeltaCAT features

deltacat/tests/experimental/catalog/iceberg/test_iceberg_catalog.py

@@ -0,0 +1,71 @@
+import tempfile
+import shutil
+import uuid
+import deltacat
+import pytest
+from deltacat import Field, Schema
+from pyiceberg.catalog import CatalogType
+
+import pyarrow as pa
+
+from deltacat.experimental.catalog.iceberg import IcebergCatalogConfig
+
+
+@pytest.fixture
+def schema_a():
+    return Schema.of(
+        [
+            Field.of(
+                field=pa.field("col1", pa.int32(), nullable=False),
+                field_id=1,
+                is_merge_key=True,
+            )
+        ]
+    )
+
+
+class TestIcebergCatalogInitialization:
+    temp_dir = None
+
+    @classmethod
+    def setup_class(cls):
+        cls.temp_dir = tempfile.mkdtemp()
+
+    @classmethod
+    def teardown_class(cls):
+        shutil.rmtree(cls.temp_dir)
+
+    def test_iceberg_catalog_and_table_create(self, schema_a):
+
+        # Register a random catalog name to avoid concurrent test conflicts
+        catalog_name = str(uuid.uuid4())
+
+        config = IcebergCatalogConfig(
+            type=CatalogType.SQL,
+            properties={
+                "warehouse": self.temp_dir,
+                "uri": f"sqlite:////{self.temp_dir}/sql-catalog.db",
+            },
+        )
+
+        # Initialize with the PyIceberg catalog
+        catalog = deltacat.IcebergCatalog.from_config(config)
+        deltacat.init(
+            {catalog_name: catalog},
+            force=True,
+        )
+
+        table_def = deltacat.create_table(
+            "test_table", catalog=catalog_name, schema=schema_a
+        )
+
+        # Fetch table we just created
+        fetched_table_def = deltacat.get_table("test_table", catalog=catalog_name)
+        assert table_def.table_version == fetched_table_def.table_version
+
+        # For now, just check that we created a table version with an equivalent schema
+        assert table_def.table_version.schema.equivalent_to(schema_a)
+
+        # Sanity check that list namespaces works
+        namespaces = deltacat.list_namespaces(catalog=catalog_name).all_items()
+        assert table_def.table.namespace in [n.namespace for n in namespaces]

deltacat/tests/experimental/compatibility/__init__.py

@@ -0,0 +1 @@
+# Test package for compatibility utilities