deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/storage/model/transform.py

@@ -1,11 +1,23 @@
 from __future__ import annotations
-from typing import List
+
+import base64
 from enum import Enum
+from typing import Dict, Any, Optional
+import pyarrow as pa
+
+from deltacat.constants import METAFILE_FORMAT, METAFILE_FORMAT_JSON
 
 
 class TransformName(str, Enum):
     IDENTITY = "identity"
     BUCKET = "bucket"
+    YEAR = "year"
+    MONTH = "month"
+    DAY = "day"
+    HOUR = "hour"
+    TRUNCATE = "truncate"
+    VOID = "void"
+    UNKNOWN = "unknown"
 
 
 class TransformParameters(dict):
@@ -17,63 +29,42 @@ class TransformParameters(dict):
     pass
 
 
-class IdentityTransformParameters(TransformParameters):
+class BucketingStrategy(str, Enum):
     """
-    This class is used to pass parameters to the identity transform
+    A bucketing strategy for the transform
     """
 
-    @staticmethod
-    def of(column_name: str) -> IdentityTransformParameters:
-        identify_transform_parameters = IdentityTransformParameters()
-        identify_transform_parameters["columnName"] = column_name
-        return identify_transform_parameters
-
-    @property
-    def column_name(self) -> str:
-        """
-        The name of the column to use for identity transform
-        """
-        return self["columnName"]
+    # Default DeltaCAT SHA-1 based hash bucketing strategy.
+    DEFAULT = "default"
 
-    @column_name.setter
-    def column_name(self, value: str) -> None:
-        self["columnName"] = value
+    # Iceberg-compliant murmur3 based hash bucketing strategy.
+    ICEBERG = "iceberg"
 
 
-class BucketingStrategy(str, Enum):
+class TruncateStrategy(str, Enum):
     """
-    A bucketing strategy for the transform
+    A truncation strategy for the transform
     """
 
-    # Uses default deltacat bucketing strategy.
-    # This strategy supports hashing on composite keys
-    # and uses SHA1 hashing for determining the bucket.
-    # If no columns passed, it will use a random UUID
-    # for determining the bucket.
+    # Default DeltaCAT truncate strategy.
     DEFAULT = "default"
 
-    # Uses iceberg compliant bucketing strategy.
-    # As indicated in the iceberg spec, it does not support
-    # composite keys and uses murmur3 hash for determining
-    # the bucket.
-    # See https://iceberg.apache.org/spec/#partitioning
+    # Iceberg-compliant truncate strategy.
     ICEBERG = "iceberg"
 
 
 class BucketTransformParameters(TransformParameters):
     """
-    Encapsulates parameters for the bucket transform.
+    Parameters for the bucket transform.
     """
 
+    @staticmethod
     def of(
-        self,
         num_buckets: int,
-        column_names: List[str],
-        bucketing_strategy: BucketingStrategy,
+        bucketing_strategy: BucketingStrategy = BucketingStrategy.DEFAULT,
     ) -> BucketTransformParameters:
         bucket_transform_parameters = BucketTransformParameters()
         bucket_transform_parameters["numBuckets"] = num_buckets
-        bucket_transform_parameters["columnNames"] = column_names
         bucket_transform_parameters["bucketingStrategy"] = bucketing_strategy
 
         return bucket_transform_parameters
@@ -81,47 +72,272 @@ class BucketTransformParameters(TransformParameters):
     @property
     def num_buckets(self) -> int:
         """
-        The total number of buckets to create for values of the column
+        The total number of buckets to create.
         """
         return self["numBuckets"]
 
     @property
-    def column_names(self) -> List[str]:
+    def bucketing_strategy(self) -> BucketingStrategy:
+        """
+        The bucketing strategy to use.
+        """
+        return BucketingStrategy(self["bucketingStrategy"])
+
+
+class TruncateTransformParameters(TransformParameters):
+    """
+    Parameters for the truncate transform.
+    """
+
+    @staticmethod
+    def of(
+        width: int,
+        truncate_strategy: TruncateStrategy = TruncateStrategy.DEFAULT,
+    ) -> TruncateTransformParameters:
+        truncate_transform_parameters = TruncateTransformParameters()
+        truncate_transform_parameters["width"] = width
+        truncate_transform_parameters["truncateStrategy"] = truncate_strategy
+        return truncate_transform_parameters
+
+    @property
+    def width(self) -> int:
         """
-        An ordered list of unique column names from the table schema
-        to use for bucketings.
+        The width to truncate the field to.
         """
-        return self["columnNames"]
+        return self["width"]
 
     @property
-    def bucketing_strategy(self) -> BucketingStrategy:
+    def truncate_strategy(self) -> TruncateStrategy:
         """
-        The bucketing strategy to used.
+        The truncate strategy to use.
         """
-        return self["bucketingStrategy"]
+        return TruncateStrategy(self["truncateStrategy"])
 
 
 class Transform(dict):
     """
-    A transform is represents how a particular column value can be
-    transformed into a new value. This is mostly used in the context
-    of partitioning the data files in a table.
+    A transform represents how a particular column value can be
+    transformed into a new value. For example, transforms may be used
+    to determine partition or sort values for table records.
+    """
+
+    @property
+    def name(self) -> TransformName:
+        return TransformName(self["name"])
+
+    @name.setter
+    def name(self, name: TransformName) -> None:
+        self["name"] = name
+
+    @property
+    def parameters(self) -> Optional[TransformParameters]:
+        return NAME_TO_TRANSFORM[self.name].parameters
+
+    @parameters.setter
+    def parameters(
+        self,
+        parameters: Optional[TransformParameters] = None,
+    ) -> None:
+        NAME_TO_TRANSFORM[self.name].parameters = parameters
+
+    @property
+    def return_type(self) -> Optional[pa.DataType]:
+        """
+        The PyArrow data type that this transform returns.
+        A return value of "None" indicates that the return type is the same
+        as the source type. Transforms that always return null return pa.null().
+        """
+        return_type = self.get("return_type")
+        if return_type is not None:
+            schema_bytes = (
+                base64.b64decode(return_type)
+                if METAFILE_FORMAT == METAFILE_FORMAT_JSON
+                else return_type
+            )
+            return_type = pa.ipc.read_schema(
+                pa.py_buffer(schema_bytes),
+            )[0].type
+        return return_type
+
+    @return_type.setter
+    def return_type(self, return_type: pa.Schema) -> None:
+        """
+        Set the PyArrow data type that this transform returns.
+        """
+        self["return_type"] = return_type.serialize().to_pybytes()
+
+    @property
+    def is_multi_field_transform(self) -> bool:
+        """
+        Whether this transform is a multi-field transform.
+        """
+        return False
+
+
+class BucketTransform(Transform):
+    """
+    A transform that hashes field values into a fixed number of buckets.
+    Returns a PyArrow int32() type.
     """
 
     @staticmethod
-    def of(
-        name: TransformName,
-        parameters: TransformParameters,
-    ) -> Transform:
-        partition_transform = Transform()
-        partition_transform["name"] = name
-        partition_transform["parameters"] = parameters
-        return partition_transform
+    def of(parameters: BucketTransformParameters) -> BucketTransform:
+        transform = BucketTransform()
+        transform.name = TransformName.BUCKET
+        transform.parameters = parameters
+        transform.return_type = pa.schema([("return_type", pa.int32())])
+        return transform
 
     @property
-    def name(self) -> TransformName:
-        return self["name"]
+    def parameters(self) -> BucketTransformParameters:
+        val: Dict[str, Any] = self.get("parameters")
+        if val is not None and not isinstance(val, BucketTransformParameters.__class__):
+            self["parameters"] = val = BucketTransformParameters(val)
+        return val
+
+    @parameters.setter
+    def parameters(
+        self,
+        parameters: Optional[BucketTransformParameters] = None,
+    ) -> None:
+        self["parameters"] = parameters
 
     @property
-    def parameters(self) -> TransformParameters:
-        return self["parameters"]
+    def is_multi_field_transform(self) -> bool:
+        return True
+
+
+class TruncateTransform(Transform):
+    """
+    A transform that truncates field values to a fixed width.
+    Returns the same type as the input field.
+    """
+
+    @staticmethod
+    def of(parameters: TruncateTransformParameters) -> TruncateTransform:
+        transform = TruncateTransform()
+        transform.name = TransformName.TRUNCATE
+        transform.parameters = parameters
+        return transform
+
+    @property
+    def parameters(self) -> TruncateTransformParameters:
+        val: Dict[str, Any] = self.get("parameters")
+        if val is not None and not isinstance(val, TruncateTransformParameters):
+            self["parameters"] = val = TruncateTransformParameters(val)
+        return val
+
+    @parameters.setter
+    def parameters(
+        self,
+        parameters: Optional[TruncateTransformParameters] = None,
+    ) -> None:
+        self["parameters"] = parameters
+
+
+class IdentityTransform(Transform):
+    """
+    A no-op transform that returns unmodified field values.
+    Returns the same PyArrow type as the input.
+    """
+
+    @staticmethod
+    def of() -> IdentityTransform:
+        transform = IdentityTransform()
+        transform.name = TransformName.IDENTITY
+        return transform
+
+
+class HourTransform(Transform):
+    """
+    A transform that returns the hour of a datetime value.
+    Returns a PyArrow int32 type representing the hour (0-23).
+    """
+
+    @staticmethod
+    def of() -> HourTransform:
+        transform = HourTransform()
+        transform.name = TransformName.HOUR
+        transform.return_type = pa.schema([("return_type", pa.int32())])
+        return transform
+
+
+class DayTransform(Transform):
+    """
+    A transform that returns the day of a datetime value.
+    Returns a PyArrow int32 type representing the day (1-31).
+    """
+
+    @staticmethod
+    def of() -> DayTransform:
+        transform = DayTransform()
+        transform.name = TransformName.DAY
+        transform.return_type = pa.schema([("return_type", pa.int32())])
+        return transform
+
+
+class MonthTransform(Transform):
+    """
+    A transform that returns the month of a datetime value.
+    Returns a PyArrow int32 type representing the month (1-12).
+    """
+
+    @staticmethod
+    def of() -> MonthTransform:
+        transform = MonthTransform()
+        transform.name = TransformName.MONTH
+        transform.return_type = pa.schema([("return_type", pa.int32())])
+        return transform
+
+
+class YearTransform(Transform):
+    """
+    A transform that returns the year of a datetime value.
+    Returns a PyArrow int32 type representing the year.
+    """
+
+    @staticmethod
+    def of() -> YearTransform:
+        transform = YearTransform()
+        transform.name = TransformName.YEAR
+        transform.return_type = pa.schema([("return_type", pa.int32())])
+        return transform
+
+
+class VoidTransform(Transform):
+    """
+    A transform that coerces all field values to None.
+    Returns a PyArrow null type.
+    """
+
+    @staticmethod
+    def of() -> VoidTransform:
+        transform = VoidTransform()
+        transform.name = TransformName.VOID
+        transform.return_type = pa.schema([("return_type", pa.null())])
+        return transform
+
+
+class UnknownTransform(Transform):
+    """
+    An unknown or invalid transform.
+    """
+
+    @staticmethod
+    def of() -> UnknownTransform:
+        transform = UnknownTransform()
+        transform.name = TransformName.UNKNOWN
+        return transform
+
+
+NAME_TO_TRANSFORM: Dict[TransformName, Transform] = {
+    TransformName.IDENTITY: IdentityTransform,
+    TransformName.BUCKET: BucketTransform,
+    TransformName.YEAR: YearTransform,
+    TransformName.MONTH: MonthTransform,
+    TransformName.DAY: DayTransform,
+    TransformName.HOUR: HourTransform,
+    TransformName.TRUNCATE: TruncateTransform,
+    TransformName.VOID: VoidTransform,
+    TransformName.UNKNOWN: UnknownTransform,
+}

deltacat/storage/model/types.py

@@ -1,16 +1,41 @@
+from __future__ import annotations
+
 from enum import Enum
 from typing import List, Union
 
-from pyarrow.parquet import ParquetFile
 import numpy as np
 import pandas as pd
 import pyarrow as pa
-from ray.data.dataset import Dataset
+import polars as pl
+from ray.data.dataset import Dataset as RayDataset
 from daft import DataFrame as DaftDataFrame
 
-LocalTable = Union[pa.Table, pd.DataFrame, np.ndarray, ParquetFile]
-LocalDataset = List[LocalTable]
-DistributedDataset = Union[Dataset, DaftDataFrame]
+from deltacat.constants import (
+    RUNNING_TXN_DIR_NAME,
+    PAUSED_TXN_DIR_NAME,
+    FAILED_TXN_DIR_NAME,
+    SUCCESS_TXN_DIR_NAME,
+)
+
+LocalTable = Union[
+    pa.Table,
+    pd.DataFrame,
+    pl.DataFrame,
+    np.ndarray,
+    pa.parquet.ParquetFile,
+]
+LocalDataset = Union[LocalTable, List[LocalTable]]
+DistributedDataset = Union[RayDataset, DaftDataFrame]
+Dataset = Union[LocalDataset, DistributedDataset]
+
+
+class StreamFormat(str, Enum):
+    DELTACAT = "deltacat"
+    ICEBERG = "iceberg"
+    HIVE = "hive"
+    HUDI = "hudi"
+    DELTA_LAKE = "delta_lake"
+    SQLITE3 = "SQLITE3"  # used by tests
 
 
 class DeltaType(str, Enum):
@@ -19,7 +44,81 @@ class DeltaType(str, Enum):
     DELETE = "delete"
 
 
+class TransactionOperationType(str, Enum):
+    CREATE = "create"
+    UPDATE = "update"
+    REPLACE = "replace"
+    DELETE = "delete"
+
+    READ_SIBLINGS = "read_siblings"
+    READ_CHILDREN = "read_children"
+    READ_LATEST = "read_latest"
+    READ_EXISTS = "read_exists"
+
+    @staticmethod
+    def write_operations():
+        return {
+            TransactionOperationType.CREATE,
+            TransactionOperationType.UPDATE,
+            TransactionOperationType.REPLACE,
+            TransactionOperationType.DELETE,
+        }
+
+    @staticmethod
+    def read_operations():
+        return {
+            TransactionOperationType.READ_SIBLINGS,
+            TransactionOperationType.READ_CHILDREN,
+            TransactionOperationType.READ_LATEST,
+            TransactionOperationType.READ_EXISTS,
+        }
+
+    def is_write_operation(self) -> bool:
+        return self in TransactionOperationType.write_operations()
+
+    def is_read_operation(self) -> bool:
+        return self in TransactionOperationType.read_operations()
+
+
+class TransactionStatus(str, Enum):
+    """
+    Transaction user status types. Every transaction status maps to a distinct
+    transaction log directory.
+    """
+
+    SUCCESS = "SUCCESS"
+    RUNNING = "RUNNING"
+    PAUSED = "PAUSED"
+    FAILED = "FAILED"
+
+    def dir_name(self) -> str:
+        if self == TransactionStatus.RUNNING:
+            return RUNNING_TXN_DIR_NAME
+        elif self == TransactionStatus.PAUSED:
+            return PAUSED_TXN_DIR_NAME
+        elif self == TransactionStatus.FAILED:
+            return FAILED_TXN_DIR_NAME
+        elif self == TransactionStatus.SUCCESS:
+            return SUCCESS_TXN_DIR_NAME
+
+
+class TransactionState(str, Enum):
+    """
+    Transaction system state types. Transaction states do not map to distinct transaction log directories,
+    but can be inferred by its presence in one or more directories. These states are used to infer whether
+    to run system activities like transaction cleanup jobs.
+    """
+
+    FAILED = "FAILED"
+    PURGED = "PURGED"
+    TIMEOUT = "TIMEOUT"
+    RUNNING = "RUNNING"
+    SUCCESS = "SUCCESS"
+    PAUSED = "PAUSED"
+
+
 class LifecycleState(str, Enum):
+    CREATED = "created"
     UNRELEASED = "unreleased"
     ACTIVE = "active"
     DEPRECATED = "deprecated"
@@ -35,22 +134,45 @@ class CommitState(str, Enum):
 
 class SchemaConsistencyType(str, Enum):
     """
-    Schemas are optional for DeltaCAT tables and can be used to inform the data
-    consistency checks run for each field. If a schema is present, it can be
-    used to enforce the following column-level data consistency policies at
-    table load time:
+    DeltaCAT table schemas can be used to inform the data consistency checks
+    run for each field. When present, the schema can be used to enforce the
+    following field-level data consistency policies at table load time:
 
-    NONE: No consistency checks are run. May be mixed with the below two
-    policies by specifying column names to pass through together with
-    column names to coerce/validate.
+    NONE: No consistency checks are run.
 
-    COERCE: Coerce fields to fit the schema whenever possible. An explicit
-    subset of column names to coerce may optionally be specified.
+    COERCE: Coerce fields to fit the schema whenever possible.
 
-    VALIDATE: Raise an error for any fields that don't fit the schema. An
-    explicit subset of column names to validate may optionally be specified.
+    VALIDATE: Raise an error for any fields that don't fit the schema.
     """
 
     NONE = "none"
     COERCE = "coerce"
     VALIDATE = "validate"
+
+
+class SortOrder(str, Enum):
+    ASCENDING = "ascending"
+    DESCENDING = "descending"
+
+    @classmethod
+    def _missing_(cls, value: str):
+        # pyiceberg.table.sorting.SortDirection mappings
+        if value.lower() == "asc":
+            return SortOrder.ASCENDING
+        elif value.lower() == "desc":
+            return SortOrder.DESCENDING
+        return None
+
+
+class NullOrder(str, Enum):
+    AT_START = "at_start"
+    AT_END = "at_end"
+
+    @classmethod
+    def _missing_(cls, value: str):
+        # pyiceberg.table.sorting.NullOrder mappings
+        if value.lower() == "nulls-first":
+            return NullOrder.AT_START
+        elif value.lower() == "nulls-last":
+            return NullOrder.AT_END
+        return None

deltacat/storage/util/scan_planner.py

@@ -0,0 +1,26 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from deltacat.storage.model.scan.push_down import Pushdown
+from deltacat.storage.model.scan.scan_plan import ScanPlan
+
+
+class ScanPlanner(ABC):
+    @abstractmethod
+    def create_scan_plan(
+        self,
+        table_name: str,
+        namespace: Optional[str] = None,
+        pushdown: Optional[Pushdown] = None,
+    ) -> ScanPlan:
+        """Return a ScanPlan for a given DeltaCAT Table after applying pushdown predicates
+
+        Args:
+            table: Name of the table
+            namespace: Optional namespace of the table. Uses default namespace if not specified.
+            pushdown: Pushdown predicates used to filter partitions/data files
+
+        Returns:
+            a ScanPlan object containing list of ScanTasks
+        """
+        pass

deltacat/tests/_io/__init__.py

@@ -0,0 +1 @@
+# NOTE - this module is renamed because it is shadowing the stdlib io module when running tests in Pycharm

deltacat/tests/{io → _io}/test_cloudpickle_bug_fix.py

@@ -45,7 +45,11 @@ class TestCloudpickleBugFix(unittest.TestCase):
     def test_sanity(self):
         ray.init(local_mode=True, ignore_reinit_error=True)
 
-        result = ray.get(calculate_pickled_length.remote(AnyObject()))
-
-        self.assertTrue(result[0] < 1000)
-        self.assertTrue(result[1] >= 5000000)
+        try:
+            result = ray.get(calculate_pickled_length.remote(AnyObject()))
+
+            self.assertTrue(result[0] < 1000)
+            self.assertTrue(result[1] >= 5000000)
+        finally:
+            if ray.is_initialized():
+                ray.shutdown()

deltacat/tests/aws/test_s3u.py

@@ -2,9 +2,8 @@ import unittest
 
 import botocore
 
-from deltacat.aws.constants import RETRYABLE_TRANSIENT_ERRORS
-from deltacat.aws.s3u import UuidBlockWritePathProvider, CapturedBlockWritePaths
-
+from deltacat.constants import RETRYABLE_TRANSIENT_ERRORS
+from deltacat.types.tables import CapturedBlockWritePaths, UuidBlockWritePathProvider
 
 import os
 from unittest import mock
@@ -99,34 +98,6 @@ class TestDownloadUpload(unittest.TestCase):
 
         assert mock_s3.put_object.call_count > 3
 
-    @patch("deltacat.aws.s3u.UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY", 1)
-    @patch("deltacat.aws.s3u.ManifestEntry")
-    @patch("deltacat.aws.s3u._get_metadata")
-    @patch("deltacat.aws.s3u.CapturedBlockWritePaths")
-    def test_upload_sliced_table_retry(
-        self,
-        mock_captured_block_write_paths,
-        mock_get_metadata,
-        mock_manifest_entry,
-    ):
-        mock_manifest_entry.from_s3_obj_url.side_effect = OSError(
-            "Please reduce your request rate.."
-        )
-        mock_get_metadata.return_value = [mock.MagicMock()]
-        cbwp = CapturedBlockWritePaths()
-        cbwp._write_paths = ["s3_write_path"]
-        cbwp._block_refs = [mock.MagicMock()]
-        mock_captured_block_write_paths.return_value = cbwp
-        with pytest.raises(RetryError):
-            s3u.upload_sliced_table(
-                mock.MagicMock(),
-                "s3-prefix",
-                mock.MagicMock(),
-                mock.MagicMock(),
-                mock.MagicMock(),
-                mock.MagicMock(),
-            )
-
     @patch("deltacat.aws.s3u.UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY", 1)
     @patch("deltacat.aws.s3u.s3_client_cache")
     def test_upload_transient_error_retry(self, mock_s3_client_cache):