deltacat 1.1.9__tar.gz → 1.1.11__tar.gz

deltacat-1.1.11/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.1
+Name: deltacat
+Version: 1.1.11
+Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
+Home-page: https://github.com/ray-project/deltacat
+Author: Ray Team
+License: UNKNOWN
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# DeltaCAT
+
+DeltaCAT is a Pythonic Data Catalog powered by Ray.
+
+Its data storage model allows you to define and manage fast, scalable,
+ACID-compliant data catalogs through git-like stage/commit APIs, and has been
+used to successfully host exabyte-scale enterprise data lakes.
+
+DeltaCAT uses the Ray distributed compute framework together with Apache Arrow
+for common table management tasks, including petabyte-scale
+change-data-capture, data consistency checks, and table repair.
+
+## Getting Started
+
+### Install
+
+```
+pip install deltacat
+```
+
+### Running Tests
+
+```
+pip3 install virtualenv
+virtualenv test_env
+source test_env/bin/activate
+pip3 install -r requirements.txt
+
+pytest
+```
+
+

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.9"
+__version__ = "1.1.11"
 
 
 __all__ = [

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/aws/redshift/model/manifest.py

@@ -99,6 +99,8 @@ class Manifest(dict):
         total_source_content_length = 0
         content_type = None
         content_encoding = None
+        partition_values_set = set()
+        partition_values = None
         if entries:
             content_type = entries[0].meta.content_type
             content_encoding = entries[0].meta.content_encoding
@@ -127,6 +129,12 @@ class Manifest(dict):
                 total_record_count += meta.record_count or 0
                 total_content_length += meta.content_length or 0
                 total_source_content_length += meta.source_content_length or 0
+                if len(partition_values_set) <= 1:
+                    partition_values_set.add(entry.meta.partition_values)
+
+        if len(partition_values_set) == 1:
+            partition_values = partition_values_set.pop()
+
         meta = ManifestMeta.of(
             total_record_count,
             total_content_length,
@@ -134,6 +142,7 @@ class Manifest(dict):
             content_encoding,
             total_source_content_length,
             entry_type=entry_type,
+            partition_values=partition_values,
         )
         manifest = Manifest._build_manifest(meta, entries, author, uuid, entry_type)
         return manifest
@@ -185,6 +194,7 @@ class ManifestMeta(dict):
         credentials: Optional[Dict[str, str]] = None,
         content_type_parameters: Optional[List[Dict[str, str]]] = None,
         entry_type: Optional[EntryType] = None,
+        partition_values: Optional[List[str]] = None,
     ) -> ManifestMeta:
         manifest_meta = ManifestMeta()
         if record_count is not None:
@@ -203,6 +213,8 @@ class ManifestMeta(dict):
             manifest_meta["credentials"] = credentials
         if entry_type is not None:
             manifest_meta["entry_type"] = entry_type.value
+        if partition_values is not None:
+            manifest_meta["partition_values"] = partition_values
         return manifest_meta
 
     @property
@@ -244,6 +256,10 @@ class ManifestMeta(dict):
             return EntryType(self["entry_type"])
         return val
 
+    @property
+    def partition_values(self) -> Optional[List[str]]:
+        return self.get("partition_values")
+
 
 class ManifestAuthor(dict):
     @staticmethod

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/aws/s3u.py

@@ -21,7 +21,7 @@ from boto3.resources.base import ServiceResource
 from botocore.client import BaseClient
 from botocore.exceptions import ClientError
 from ray.data.block import Block, BlockAccessor, BlockMetadata
-from ray.data.datasource import BlockWritePathProvider
+from ray.data.datasource import FilenameProvider
 from ray.types import ObjectRef
 from tenacity import (
     Retrying,
@@ -70,9 +70,6 @@ from deltacat.exceptions import categorize_errors
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-# TODO(raghumdani): refactor redshift datasource to reuse the
-# same module for writing output files.
-
 
 class CapturedBlockWritePaths:
     def __init__(self):
@@ -100,12 +97,15 @@ class CapturedBlockWritePaths:
         return self._block_refs
 
 
-class UuidBlockWritePathProvider(BlockWritePathProvider):
+class UuidBlockWritePathProvider(FilenameProvider):
     """Block write path provider implementation that writes each
     dataset block out to a file of the form: {base_path}/{uuid}
     """
 
-    def __init__(self, capture_object: CapturedBlockWritePaths):
+    def __init__(
+        self, capture_object: CapturedBlockWritePaths, base_path: Optional[str] = None
+    ):
+        self.base_path = base_path
         self.write_paths: List[str] = []
         self.block_refs: List[ObjectRef[Block]] = []
         self.capture_object = capture_object
@@ -117,6 +117,19 @@ class UuidBlockWritePathProvider(BlockWritePathProvider):
                 self.block_refs,
             )
 
+    def get_filename_for_block(
+        self, block: Any, task_index: int, block_index: int
+    ) -> str:
+        if self.base_path is None:
+            raise ValueError(
+                "Base path must be provided to UuidBlockWritePathProvider",
+            )
+        return self._get_write_path_for_block(
+            base_path=self.base_path,
+            block=block,
+            block_index=block_index,
+        )
+
     def _get_write_path_for_block(
         self,
         base_path: str,
@@ -143,13 +156,6 @@ class UuidBlockWritePathProvider(BlockWritePathProvider):
         block_index: Optional[int] = None,
         file_format: Optional[str] = None,
     ) -> str:
-        """
-        TODO: BlockWritePathProvider is deprecated as of Ray version 2.20.0. Please use FilenameProvider.
-        See: https://docs.ray.io/en/master/data/api/doc/ray.data.datasource.FilenameProvider.html
-        Also See: https://github.com/ray-project/deltacat/issues/299
-
-        Hence, this class only works with Ray version 2.20.0 or lower when used in Ray Dataset.
-        """
         return self._get_write_path_for_block(
             base_path,
             filesystem=filesystem,

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/compute/compactor/compaction_session.py

@@ -193,6 +193,7 @@ def compact_partition(
             round_completion_file_s3_url = rcf.write_round_completion_file(
                 compaction_artifact_s3_bucket,
                 new_rcf_partition_locator,
+                partition.locator,
                 new_rci,
                 **s3_client_kwargs,
             )
@@ -312,7 +313,10 @@ def _execute_compaction_round(
     round_completion_info = None
     if not rebase_source_partition_locator:
         round_completion_info = rcf.read_round_completion_file(
-            compaction_artifact_s3_bucket, source_partition_locator, **s3_client_kwargs
+            compaction_artifact_s3_bucket,
+            source_partition_locator,
+            destination_partition_locator,
+            **s3_client_kwargs,
         )
         if not round_completion_info:
             logger.info(

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/compute/compactor/repartition_session.py

@@ -177,6 +177,7 @@ def repartition(
         s3_client_kwargs = {}
 
     return rcf.write_round_completion_file(
+        None,
         None,
         None,
         repartition_completion_info,

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/compute/compactor/utils/round_completion_file.py

@@ -12,10 +12,17 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 def get_round_completion_file_s3_url(
-    bucket: str, source_partition_locator: PartitionLocator
+    bucket: str,
+    source_partition_locator: PartitionLocator,
+    destination_partition_locator: Optional[PartitionLocator] = None,
 ) -> str:
 
     base_url = source_partition_locator.path(f"s3://{bucket}")
+    if destination_partition_locator:
+        base_url = destination_partition_locator.path(
+            f"s3://{bucket}/{source_partition_locator.hexdigest()}"
+        )
+
     return f"{base_url}.json"
 
 
@@ -23,20 +30,41 @@ def get_round_completion_file_s3_url(
 def read_round_completion_file(
     bucket: str,
     source_partition_locator: PartitionLocator,
+    destination_partition_locator: Optional[PartitionLocator] = None,
     **s3_client_kwargs: Optional[Dict[str, Any]],
 ) -> RoundCompletionInfo:
 
-    round_completion_file_url = get_round_completion_file_s3_url(
+    all_uris = []
+    if destination_partition_locator:
+        round_completion_file_url_with_destination = get_round_completion_file_s3_url(
+            bucket,
+            source_partition_locator,
+            destination_partition_locator,
+        )
+        all_uris.append(round_completion_file_url_with_destination)
+
+    # Note: we read from RCF at two different URI for backward
+    # compatibility reasons.
+    round_completion_file_url_prev = get_round_completion_file_s3_url(
         bucket,
         source_partition_locator,
     )
-    logger.info(f"reading round completion file from: {round_completion_file_url}")
+
+    all_uris.append(round_completion_file_url_prev)
+
     round_completion_info = None
-    result = s3_utils.download(round_completion_file_url, False, **s3_client_kwargs)
-    if result:
-        json_str = result["Body"].read().decode("utf-8")
-        round_completion_info = RoundCompletionInfo(json.loads(json_str))
-        logger.info(f"read round completion info: {round_completion_info}")
+
+    for rcf_uri in all_uris:
+        logger.info(f"Reading round completion file from: {rcf_uri}")
+        result = s3_utils.download(rcf_uri, False, **s3_client_kwargs)
+        if result:
+            json_str = result["Body"].read().decode("utf-8")
+            round_completion_info = RoundCompletionInfo(json.loads(json_str))
+            logger.info(f"Read round completion info: {round_completion_info}")
+            break
+        else:
+            logger.warn(f"Round completion file not present at {rcf_uri}")
+
     return round_completion_info
 
 
@@ -44,8 +72,9 @@ def read_round_completion_file(
 def write_round_completion_file(
     bucket: Optional[str],
     source_partition_locator: Optional[PartitionLocator],
+    destination_partition_locator: Optional[PartitionLocator],
     round_completion_info: RoundCompletionInfo,
-    completion_file_s3_url: str = None,
+    completion_file_s3_url: Optional[str] = None,
     **s3_client_kwargs: Optional[Dict[str, Any]],
 ) -> str:
     if bucket is None and completion_file_s3_url is None:
@@ -56,6 +85,7 @@ def write_round_completion_file(
         completion_file_s3_url = get_round_completion_file_s3_url(
             bucket,
             source_partition_locator,
+            destination_partition_locator,
         )
     logger.info(f"writing round completion file to: {completion_file_s3_url}")
     s3_utils.upload(

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/compute/compactor_v2/compaction_session.py

@@ -24,7 +24,7 @@ from deltacat.compute.compactor import (
 )
 from deltacat.compute.compactor_v2.model.merge_result import MergeResult
 from deltacat.compute.compactor_v2.model.hash_bucket_result import HashBucketResult
-from deltacat.compute.compactor_v2.model.compaction_session import (
+from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
     ExecutionCompactionResult,
 )
 from deltacat.compute.compactor.model.materialize_result import MaterializeResult
@@ -78,6 +78,7 @@ from deltacat.compute.compactor_v2.utils.task_options import (
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 from deltacat.exceptions import categorize_errors
+from deltacat.compute.compactor_v2.constants import COMPACT_PARTITION_METRIC_PREFIX
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -86,7 +87,7 @@ if importlib.util.find_spec("memray"):
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-@metrics
+@metrics(prefix=COMPACT_PARTITION_METRIC_PREFIX)
 @categorize_errors
 def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
     assert (
@@ -109,7 +110,6 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
             f"Partition-{params.source_partition_locator} -> "
             f"{compaction_session_type} Compaction session data processing completed"
         )
-        round_completion_file_s3_url: Optional[str] = None
         if execute_compaction_result.new_compacted_partition:
             previous_partition: Optional[Partition] = None
             if execute_compaction_result.is_inplace_compacted:
@@ -131,19 +131,13 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
                 **params.deltacat_storage_kwargs,
             )
             logger.info(f"Committed compacted partition: {committed_partition}")
-            round_completion_file_s3_url = rcf.write_round_completion_file(
-                params.compaction_artifact_s3_bucket,
-                execute_compaction_result.new_round_completion_file_partition_locator,
-                execute_compaction_result.new_round_completion_info,
-                **params.s3_client_kwargs,
-            )
         else:
             logger.warning("No new partition was committed during compaction.")
 
         logger.info(
             f"Completed compaction session for: {params.source_partition_locator}"
         )
-        return round_completion_file_s3_url
+        return execute_compaction_result.round_completion_file_s3_url
 
 
 def _execute_compaction(
@@ -188,6 +182,7 @@ def _execute_compaction(
         round_completion_info = rcf.read_round_completion_file(
             params.compaction_artifact_s3_bucket,
             params.source_partition_locator,
+            params.destination_partition_locator,
             **params.s3_client_kwargs,
         )
         if not round_completion_info:
@@ -684,9 +679,18 @@ def _execute_compaction(
             f"and rcf source partition_id of {rcf_source_partition_locator.partition_id}."
         )
         rcf_source_partition_locator = compacted_partition.locator
+
+    round_completion_file_s3_url = rcf.write_round_completion_file(
+        params.compaction_artifact_s3_bucket,
+        rcf_source_partition_locator,
+        compacted_partition.locator,
+        new_round_completion_info,
+        **params.s3_client_kwargs,
+    )
+
     return ExecutionCompactionResult(
         compacted_partition,
         new_round_completion_info,
-        rcf_source_partition_locator,
+        round_completion_file_s3_url,
         is_inplace_compacted,
     )

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/compute/compactor_v2/constants.py

@@ -68,3 +68,6 @@ DISCOVER_DELTAS_METRIC_PREFIX = "discover_deltas"
 
 # Metric prefix for prepare deletes
 PREPARE_DELETES_METRIC_PREFIX = "prepare_deletes"
+
+# Metric prefix for compact partition method
+COMPACT_PARTITION_METRIC_PREFIX = "compact_partition"

deltacat-1.1.9/deltacat/compute/compactor_v2/model/compaction_session.py → deltacat-1.1.11/deltacat/compute/compactor_v2/model/evaluate_compaction_result.py

@@ -2,7 +2,6 @@ from dataclasses import dataclass, fields
 
 from deltacat.storage import (
     Partition,
-    PartitionLocator,
 )
 from deltacat.compute.compactor import (
     RoundCompletionInfo,
@@ -14,7 +13,7 @@ from typing import Optional
 class ExecutionCompactionResult:
     new_compacted_partition: Optional[Partition]
     new_round_completion_info: Optional[RoundCompletionInfo]
-    new_round_completion_file_partition_locator: Optional[PartitionLocator]
+    round_completion_file_s3_url: Optional[str]
     is_inplace_compacted: bool
 
     def __iter__(self):

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -162,7 +162,7 @@ def group_by_pk_hash_bucket(
         len(new_tables) == 1
     ), f"Expected only 1 table in the result but found {len(new_tables)}"
 
-    table = generate_pk_hash_column([table], primary_keys, requires_hash=True)[0]
+    table = new_tables[0]
 
     # group hash bucket record indices
     result = group_record_indices_by_hash_bucket(

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/exceptions.py

@@ -213,11 +213,14 @@ def categorize_errors(func: Callable):
         except BaseException as e:
             deltacat_storage = None
             deltacat_storage_kwargs = {}
+            all_args = args
             if kwargs:
                 deltacat_storage = kwargs.get(DELTACAT_STORAGE_PARAM)
                 deltacat_storage_kwargs = kwargs.get(DELTACAT_STORAGE_KWARGS_PARAM, {})
-            if not deltacat_storage and args:
-                for arg in args:
+                all_args = all_args + tuple(kwargs.values())
+
+            if not deltacat_storage and all_args:
+                for arg in all_args:
                     if (
                         isinstance(arg, dict)
                         and arg.get(DELTACAT_STORAGE_PARAM) is not None

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/io/dataset.py

@@ -6,9 +6,6 @@ from typing import Any, Callable, Dict, Optional, TypeVar, Union, cast
 import pyarrow as pa
 import s3fs
 from ray.data import Dataset
-from ray.data.datasource import BlockWritePathProvider, DefaultBlockWritePathProvider
-
-from deltacat.io.aws.redshift.redshift_datasource import RedshiftDatasource
 
 T = TypeVar("T")
 
@@ -27,7 +24,6 @@ class DeltacatDataset(Dataset[T]):
         filesystem: Optional[Union[pa.fs.FileSystem, s3fs.S3FileSystem]] = None,
         try_create_dir: bool = True,
         arrow_open_stream_args: Optional[Dict[str, Any]] = None,
-        block_path_provider: BlockWritePathProvider = DefaultBlockWritePathProvider(),
         arrow_parquet_args_fn: Callable[[], Dict[str, Any]] = lambda: {},
         **arrow_parquet_args,
     ) -> None:
@@ -59,9 +55,8 @@ class DeltacatDataset(Dataset[T]):
                 if True. Does nothing if all directories already exist.
             arrow_open_stream_args: kwargs passed to
                 pyarrow.fs.FileSystem.open_output_stream
-            block_path_provider: BlockWritePathProvider implementation
-                to write each dataset block to a custom output path. Uses
-                DefaultBlockWritePathProvider if None.
+            filename_provider: FilenameProvider implementation
+                to write each dataset block to a custom output path.
             arrow_parquet_args_fn: Callable that returns a dictionary of write
                 arguments to use when writing each block to a file. Overrides
                 any duplicate keys from arrow_parquet_args. This should be used
@@ -72,14 +67,7 @@ class DeltacatDataset(Dataset[T]):
                 pyarrow.parquet.write_table(), which is used to write out each
                 block to a file.
         """
-        self.write_datasource(
-            RedshiftDatasource(),
-            path=path,
-            dataset_uuid=self._uuid,
-            filesystem=filesystem,
-            try_create_dir=try_create_dir,
-            open_stream_args=arrow_open_stream_args,
-            block_path_provider=block_path_provider,
-            write_args_fn=arrow_parquet_args_fn,
-            **arrow_parquet_args,
+        raise NotImplementedError(
+            "Writing to Redshift is not yet supported. "
+            "Please use DeltacatDataset.write_parquet() instead."
         )

{deltacat-1.1.9 → deltacat-1.1.11}/deltacat/storage/__init__.py

@@ -14,6 +14,20 @@ from deltacat.storage.model.stream import Stream, StreamLocator
 from deltacat.storage.model.table import Table, TableLocator
 from deltacat.storage.model.table_version import TableVersion, TableVersionLocator
 from deltacat.storage.model.delete_parameters import DeleteParameters
+from deltacat.storage.model.partition_spec import (
+    PartitionFilter,
+    PartitionValues,
+    DeltaPartitionSpec,
+    StreamPartitionSpec,
+)
+from deltacat.storage.model.transform import (
+    Transform,
+    TransformName,
+    TransformParameters,
+    BucketingStrategy,
+    BucketTransformParameters,
+    IdentityTransformParameters,
+)
 
 from deltacat.storage.model.types import (
     CommitState,
@@ -56,4 +70,14 @@ __all__ = [
     "TableVersionLocator",
     "SortKey",
     "SortOrder",
+    "PartitionFilter",
+    "PartitionValues",
+    "DeltaPartitionSpec",
+    "StreamPartitionSpec",
+    "Transform",
+    "TransformName",
+    "TransformParameters",
+    "BucketingStrategy",
+    "BucketTransformParameters",
+    "IdentityTransformParameters",
 ]