deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/storage/model/transform.py

@@ -1,6 +1,11 @@
 from __future__ import annotations
+
+import base64
 from enum import Enum
 from typing import Dict, Any, Optional
+import pyarrow as pa
+
+from deltacat.constants import METAFILE_FORMAT, METAFILE_FORMAT_JSON
 
 
 class TransformName(str, Enum):
@@ -36,6 +41,18 @@ class BucketingStrategy(str, Enum):
     ICEBERG = "iceberg"
 
 
+class TruncateStrategy(str, Enum):
+    """
+    A truncation strategy for the transform
+    """
+
+    # Default DeltaCAT truncate strategy.
+    DEFAULT = "default"
+
+    # Iceberg-compliant truncate strategy.
+    ICEBERG = "iceberg"
+
+
 class BucketTransformParameters(TransformParameters):
     """
     Parameters for the bucket transform.
@@ -44,7 +61,7 @@ class BucketTransformParameters(TransformParameters):
     @staticmethod
     def of(
         num_buckets: int,
-        bucketing_strategy: BucketingStrategy,
+        bucketing_strategy: BucketingStrategy = BucketingStrategy.DEFAULT,
     ) -> BucketTransformParameters:
         bucket_transform_parameters = BucketTransformParameters()
         bucket_transform_parameters["numBuckets"] = num_buckets
@@ -73,9 +90,13 @@ class TruncateTransformParameters(TransformParameters):
     """
 
     @staticmethod
-    def of(width: int) -> TruncateTransformParameters:
+    def of(
+        width: int,
+        truncate_strategy: TruncateStrategy = TruncateStrategy.DEFAULT,
+    ) -> TruncateTransformParameters:
         truncate_transform_parameters = TruncateTransformParameters()
         truncate_transform_parameters["width"] = width
+        truncate_transform_parameters["truncateStrategy"] = truncate_strategy
         return truncate_transform_parameters
 
     @property
@@ -85,6 +106,13 @@ class TruncateTransformParameters(TransformParameters):
         """
         return self["width"]
 
+    @property
+    def truncate_strategy(self) -> TruncateStrategy:
+        """
+        The truncate strategy to use.
+        """
+        return TruncateStrategy(self["truncateStrategy"])
+
 
 class Transform(dict):
     """
@@ -112,10 +140,44 @@ class Transform(dict):
     ) -> None:
         NAME_TO_TRANSFORM[self.name].parameters = parameters
 
+    @property
+    def return_type(self) -> Optional[pa.DataType]:
+        """
+        The PyArrow data type that this transform returns.
+        A return value of "None" indicates that the return type is the same
+        as the source type. Transforms that always return null return pa.null().
+        """
+        return_type = self.get("return_type")
+        if return_type is not None:
+            schema_bytes = (
+                base64.b64decode(return_type)
+                if METAFILE_FORMAT == METAFILE_FORMAT_JSON
+                else return_type
+            )
+            return_type = pa.ipc.read_schema(
+                pa.py_buffer(schema_bytes),
+            )[0].type
+        return return_type
+
+    @return_type.setter
+    def return_type(self, return_type: pa.Schema) -> None:
+        """
+        Set the PyArrow data type that this transform returns.
+        """
+        self["return_type"] = return_type.serialize().to_pybytes()
+
+    @property
+    def is_multi_field_transform(self) -> bool:
+        """
+        Whether this transform is a multi-field transform.
+        """
+        return False
+
 
 class BucketTransform(Transform):
     """
     A transform that hashes field values into a fixed number of buckets.
+    Returns a PyArrow int32() type.
     """
 
     @staticmethod
@@ -123,6 +185,7 @@ class BucketTransform(Transform):
         transform = BucketTransform()
         transform.name = TransformName.BUCKET
         transform.parameters = parameters
+        transform.return_type = pa.schema([("return_type", pa.int32())])
         return transform
 
     @property
@@ -139,10 +202,15 @@ class BucketTransform(Transform):
     ) -> None:
         self["parameters"] = parameters
 
+    @property
+    def is_multi_field_transform(self) -> bool:
+        return True
+
 
 class TruncateTransform(Transform):
     """
     A transform that truncates field values to a fixed width.
+    Returns the same type as the input field.
     """
 
     @staticmethod
@@ -170,6 +238,7 @@ class TruncateTransform(Transform):
 class IdentityTransform(Transform):
     """
     A no-op transform that returns unmodified field values.
+    Returns the same PyArrow type as the input.
     """
 
     @staticmethod
@@ -182,60 +251,70 @@ class IdentityTransform(Transform):
 class HourTransform(Transform):
     """
     A transform that returns the hour of a datetime value.
+    Returns a PyArrow int32 type representing the hour (0-23).
     """
 
     @staticmethod
     def of() -> HourTransform:
         transform = HourTransform()
         transform.name = TransformName.HOUR
+        transform.return_type = pa.schema([("return_type", pa.int32())])
         return transform
 
 
 class DayTransform(Transform):
     """
     A transform that returns the day of a datetime value.
+    Returns a PyArrow int32 type representing the day (1-31).
     """
 
     @staticmethod
     def of() -> DayTransform:
         transform = DayTransform()
         transform.name = TransformName.DAY
+        transform.return_type = pa.schema([("return_type", pa.int32())])
         return transform
 
 
 class MonthTransform(Transform):
     """
     A transform that returns the month of a datetime value.
+    Returns a PyArrow int32 type representing the month (1-12).
     """
 
     @staticmethod
     def of() -> MonthTransform:
         transform = MonthTransform()
         transform.name = TransformName.MONTH
+        transform.return_type = pa.schema([("return_type", pa.int32())])
         return transform
 
 
 class YearTransform(Transform):
     """
     A transform that returns the year of a datetime value.
+    Returns a PyArrow int32 type representing the year.
     """
 
     @staticmethod
     def of() -> YearTransform:
         transform = YearTransform()
         transform.name = TransformName.YEAR
+        transform.return_type = pa.schema([("return_type", pa.int32())])
         return transform
 
 
 class VoidTransform(Transform):
     """
     A transform that coerces all field values to None.
+    Returns a PyArrow null type.
     """
 
     @staticmethod
     def of() -> VoidTransform:
         transform = VoidTransform()
         transform.name = TransformName.VOID
+        transform.return_type = pa.schema([("return_type", pa.null())])
         return transform
 
 

deltacat/storage/model/types.py

@@ -6,23 +6,33 @@ from typing import List, Union
 import numpy as np
 import pandas as pd
 import pyarrow as pa
-from ray.data.dataset import Dataset
+import polars as pl
+from ray.data.dataset import Dataset as RayDataset
 from daft import DataFrame as DaftDataFrame
 
+from deltacat.constants import (
+    RUNNING_TXN_DIR_NAME,
+    PAUSED_TXN_DIR_NAME,
+    FAILED_TXN_DIR_NAME,
+    SUCCESS_TXN_DIR_NAME,
+)
 
 LocalTable = Union[
     pa.Table,
     pd.DataFrame,
+    pl.DataFrame,
     np.ndarray,
     pa.parquet.ParquetFile,
 ]
-LocalDataset = List[LocalTable]
-DistributedDataset = Union[Dataset, DaftDataFrame]
+LocalDataset = Union[LocalTable, List[LocalTable]]
+DistributedDataset = Union[RayDataset, DaftDataFrame]
+Dataset = Union[LocalDataset, DistributedDataset]
 
 
 class StreamFormat(str, Enum):
     DELTACAT = "deltacat"
     ICEBERG = "iceberg"
+    HIVE = "hive"
     HUDI = "hudi"
     DELTA_LAKE = "delta_lake"
     SQLITE3 = "SQLITE3"  # used by tests
@@ -34,34 +44,10 @@ class DeltaType(str, Enum):
     DELETE = "delete"
 
 
-class TransactionType(str, Enum):
-    # the transaction reads existing data
-    # does not conflict with any other transaction types
-    READ = "read"
-    # the transaction only appends new data
-    # conflicts with other transaction types can be auto-resolved
-    APPEND = "append"
-    # the transaction alters existing data
-    # (even if it also appends data)
-    # conflicts with other alters/overwrites/restates/deletes fail
-    ALTER = "alter"
-    # the transaction overwrites existing data
-    # (even if it also appends or alters data)
-    # conflicts with other alters/overwrites/restates/deletes fail
-    OVERWRITE = "overwrite"
-    # the transaction restates existing data with a new layout
-    # (even if it appends, alters, or overwrites data to do so)
-    # conflicts with other alters/overwrites/restates/deletes fail
-    RESTATE = "restate"
-    # the transaction deletes existing data
-    # (even if it also appends, alters, overwrites, or restates data)
-    # conflicts with other alters/overwrites/restates/deletes fail
-    DELETE = "delete"
-
-
 class TransactionOperationType(str, Enum):
     CREATE = "create"
     UPDATE = "update"
+    REPLACE = "replace"
     DELETE = "delete"
 
     READ_SIBLINGS = "read_siblings"
@@ -74,6 +60,7 @@ class TransactionOperationType(str, Enum):
         return {
             TransactionOperationType.CREATE,
             TransactionOperationType.UPDATE,
+            TransactionOperationType.REPLACE,
             TransactionOperationType.DELETE,
         }
 
@@ -90,7 +77,44 @@ class TransactionOperationType(str, Enum):
         return self in TransactionOperationType.write_operations()
 
     def is_read_operation(self) -> bool:
-        return self in TransactionOperationType.read_operatins()
+        return self in TransactionOperationType.read_operations()
+
+
+class TransactionStatus(str, Enum):
+    """
+    Transaction user status types. Every transaction status maps to a distinct
+    transaction log directory.
+    """
+
+    SUCCESS = "SUCCESS"
+    RUNNING = "RUNNING"
+    PAUSED = "PAUSED"
+    FAILED = "FAILED"
+
+    def dir_name(self) -> str:
+        if self == TransactionStatus.RUNNING:
+            return RUNNING_TXN_DIR_NAME
+        elif self == TransactionStatus.PAUSED:
+            return PAUSED_TXN_DIR_NAME
+        elif self == TransactionStatus.FAILED:
+            return FAILED_TXN_DIR_NAME
+        elif self == TransactionStatus.SUCCESS:
+            return SUCCESS_TXN_DIR_NAME
+
+
+class TransactionState(str, Enum):
+    """
+    Transaction system state types. Transaction states do not map to distinct transaction log directories,
+    but can be inferred by its presence in one or more directories. These states are used to infer whether
+    to run system activities like transaction cleanup jobs.
+    """
+
+    FAILED = "FAILED"
+    PURGED = "PURGED"
+    TIMEOUT = "TIMEOUT"
+    RUNNING = "RUNNING"
+    SUCCESS = "SUCCESS"
+    PAUSED = "PAUSED"
 
 
 class LifecycleState(str, Enum):

deltacat/storage/util/scan_planner.py

@@ -0,0 +1,26 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from deltacat.storage.model.scan.push_down import Pushdown
+from deltacat.storage.model.scan.scan_plan import ScanPlan
+
+
+class ScanPlanner(ABC):
+    @abstractmethod
+    def create_scan_plan(
+        self,
+        table_name: str,
+        namespace: Optional[str] = None,
+        pushdown: Optional[Pushdown] = None,
+    ) -> ScanPlan:
+        """Return a ScanPlan for a given DeltaCAT Table after applying pushdown predicates
+
+        Args:
+            table: Name of the table
+            namespace: Optional namespace of the table. Uses default namespace if not specified.
+            pushdown: Pushdown predicates used to filter partitions/data files
+
+        Returns:
+            a ScanPlan object containing list of ScanTasks
+        """
+        pass

deltacat/tests/_io/test_cloudpickle_bug_fix.py

@@ -45,7 +45,11 @@ class TestCloudpickleBugFix(unittest.TestCase):
     def test_sanity(self):
         ray.init(local_mode=True, ignore_reinit_error=True)
 
-        result = ray.get(calculate_pickled_length.remote(AnyObject()))
-
-        self.assertTrue(result[0] < 1000)
-        self.assertTrue(result[1] >= 5000000)
+        try:
+            result = ray.get(calculate_pickled_length.remote(AnyObject()))
+
+            self.assertTrue(result[0] < 1000)
+            self.assertTrue(result[1] >= 5000000)
+        finally:
+            if ray.is_initialized():
+                ray.shutdown()

deltacat/tests/aws/test_s3u.py

@@ -2,9 +2,8 @@ import unittest
 
 import botocore
 
-from deltacat.aws.constants import RETRYABLE_TRANSIENT_ERRORS
-from deltacat.aws.s3u import UuidBlockWritePathProvider, CapturedBlockWritePaths
-
+from deltacat.constants import RETRYABLE_TRANSIENT_ERRORS
+from deltacat.types.tables import CapturedBlockWritePaths, UuidBlockWritePathProvider
 
 import os
 from unittest import mock
@@ -99,34 +98,6 @@ class TestDownloadUpload(unittest.TestCase):
 
         assert mock_s3.put_object.call_count > 3
 
-    @patch("deltacat.aws.s3u.UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY", 1)
-    @patch("deltacat.aws.s3u.ManifestEntry")
-    @patch("deltacat.aws.s3u._get_metadata")
-    @patch("deltacat.aws.s3u.CapturedBlockWritePaths")
-    def test_upload_sliced_table_retry(
-        self,
-        mock_captured_block_write_paths,
-        mock_get_metadata,
-        mock_manifest_entry,
-    ):
-        mock_manifest_entry.from_s3_obj_url.side_effect = OSError(
-            "Please reduce your request rate.."
-        )
-        mock_get_metadata.return_value = [mock.MagicMock()]
-        cbwp = CapturedBlockWritePaths()
-        cbwp._write_paths = ["s3_write_path"]
-        cbwp._block_refs = [mock.MagicMock()]
-        mock_captured_block_write_paths.return_value = cbwp
-        with pytest.raises(RetryError):
-            s3u.upload_sliced_table(
-                mock.MagicMock(),
-                "s3-prefix",
-                mock.MagicMock(),
-                mock.MagicMock(),
-                mock.MagicMock(),
-                mock.MagicMock(),
-            )
-
     @patch("deltacat.aws.s3u.UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY", 1)
     @patch("deltacat.aws.s3u.s3_client_cache")
     def test_upload_transient_error_retry(self, mock_s3_client_cache):

deltacat/tests/catalog/main/test_catalog_impl_namespace_operations.py

@@ -0,0 +1,130 @@
+import shutil
+
+from deltacat.catalog import get_catalog_properties
+from deltacat.exceptions import NamespaceAlreadyExistsError
+import pytest
+import tempfile
+import deltacat.catalog.main.impl as catalog
+
+
+class TestCatalogNamespaceOperations:
+    temp_dir = None
+    property_catalog = None
+    catalog = None
+
+    @classmethod
+    def setup_class(cls):
+        cls.temp_dir = tempfile.mkdtemp()
+        cls.catalog_properties = get_catalog_properties(root=cls.temp_dir)
+
+    @classmethod
+    def teardown_class(cls):
+        shutil.rmtree(cls.temp_dir)
+
+    def test_create_namespace(self):
+        """Test creating a namespace with properties"""
+        namespace = "test_create_namespace"
+        properties = {"description": "Test Namespace", "owner": "test-user"}
+
+        # Create namespace
+        catalog.create_namespace(
+            namespace=namespace, properties=properties, inner=self.catalog_properties
+        )
+
+        # Verify namespace exists
+        assert catalog.namespace_exists(namespace, inner=self.catalog_properties)
+
+        # Get namespace and verify properties
+        namespace = catalog.get_namespace(namespace, inner=self.catalog_properties)
+        assert namespace.namespace == "test_create_namespace"
+        assert namespace.properties["description"] == "Test Namespace"
+
+    def test_get_namespace(self):
+        """Test getting namespace properties"""
+        namespace = "test_get_namespace"
+        properties = {"description": "foo", "created_by": "bar"}
+
+        # Create namespace
+        catalog.create_namespace(
+            namespace=namespace, properties=properties, inner=self.catalog_properties
+        )
+
+        # Get namespace properties
+        namespace = catalog.get_namespace(namespace, inner=self.catalog_properties)
+
+        # Verify properties
+        assert namespace.namespace == "test_get_namespace"
+        assert namespace.properties["created_by"] == "bar"
+
+    def test_namespace_exists(self):
+        """Test checking if a namespace exists"""
+        existing_namespace = "test_namespace_exists"
+        non_existing_namespace = "non_existing_namespace"
+
+        # Create namespace
+        catalog.create_namespace(
+            namespace=existing_namespace, properties={}, inner=self.catalog_properties
+        )
+
+        # Check existing namespace
+        assert catalog.namespace_exists(
+            existing_namespace, inner=self.catalog_properties
+        )
+
+        # Check non-existing namespace
+        assert not catalog.namespace_exists(
+            non_existing_namespace, inner=self.catalog_properties
+        )
+
+    def test_create_namespace_already_exists(self):
+        """Test creating a namespace that already exists should fail"""
+        namespace = "test_create_namespace_already_exists"
+        properties = {"description": "Test namespace", "owner": "test-user"}
+
+        # Create namespace first time
+        catalog.create_namespace(
+            namespace=namespace,
+            properties=properties,
+            inner=self.catalog_properties,
+        )
+
+        # Verify namespace exists
+        assert catalog.namespace_exists(namespace, inner=self.catalog_properties)
+
+        # Try to create the same namespace again, should raise ValueError
+        with pytest.raises(NamespaceAlreadyExistsError, match=namespace):
+            catalog.create_namespace(
+                namespace=namespace,
+                properties=properties,
+                inner=self.catalog_properties,
+            )
+
+    def test_drop_namespace(self):
+        """Test dropping a namespace"""
+        namespace = "test_drop_namespace"
+        properties = {"description": "Test Namespace", "owner": "test-user"}
+
+        # Create namespace
+        catalog.create_namespace(
+            namespace=namespace,
+            properties=properties,
+            inner=self.catalog_properties,
+        )
+
+        # Verify namespace exists
+        assert catalog.namespace_exists(
+            namespace,
+            inner=self.catalog_properties,
+        )
+
+        # Drop namespace
+        catalog.drop_namespace(
+            namespace,
+            inner=self.catalog_properties,
+        )
+
+        # Verify namespace does not exist
+        assert not catalog.namespace_exists(
+            namespace,
+            inner=self.catalog_properties,
+        )