deltacat 2.0.0b9__py3-none-any.whl → 2.0.0b11__py3-none-any.whl

deltacat/catalog/model/properties.py

@@ -1,6 +1,9 @@
 from __future__ import annotations
+
 from typing import Optional, Any
 
+import os
+
 import pyarrow
 from deltacat.constants import DELTACAT_ROOT
 
@@ -8,18 +11,17 @@ from deltacat.utils.filesystem import resolve_path_and_filesystem
 
 
 def get_catalog_properties(
-    *args,
+    *,
     catalog: Optional[CatalogProperties] = None,
     inner: Optional[CatalogProperties] = None,
     **kwargs,
 ) -> CatalogProperties:
     """
-    Helper function to fetch CatalogProperties instance. You are meant to call this by providing your functions
-    kwargs, OR to directly pass through CatalogProperty configuration keys like "root" in kwargs.
+    Helper function to fetch CatalogProperties instance.
 
-    This will look for a CatalogProperty value in the kwargs "catalog" or "inner". If these are found, it returns
-    the CatalogProperty value under that kwarg. Otherwise, it will pass through kwargs to the CatalogProperties
-    constructor.
+    This will look first look for CatalogProperties in either "catalog"
+    or "inner" and otherwise passes all keyword arguments to the
+    CatalogProperties constructor.
     """
     properties = catalog if catalog is not None else inner
     if properties is not None and isinstance(properties, CatalogProperties):
@@ -39,21 +41,22 @@ class CatalogProperties:
     DeltaCAT catalog instance. Properties are set from system environment
     variables unless explicit overrides are provided during initialization.
 
-    Catalog and storage APIs rely on the property catalog to retrieve durable state about the catalog they're
-    working against.
+    Catalog and storage APIs rely on the property catalog to retrieve durable
+    state about the catalog they're working against.
 
     Attributes:
-        root (str): URI string The root path where catalog metadata and data
-            files are stored. Root is determined (in prededence order) by:
-            1. check kwargs for "root"
-            2. check env variable "DELTACAT_ROOT"
-            3. default to ${cwd}/.deltacat
+        root: The root path for catalog metadata and data storage. Resolved by
+            searching for the root path in the following order:
+            1. "root" constructor input argument
+            2. "DELTACAT_ROOT" system environment variable
+            3. default to "./.deltacat/"
 
         filesystem: The filesystem implementation that should be used for
             reading/writing files. If None, a filesystem will be inferred from
             the catalog root path.
 
-        storage: Storage class implementation (overrides default filesystem storage impl)
+        storage: Storage class implementation (overrides default filesystem
+            storage impl)
     """
 
     def __init__(
@@ -61,28 +64,26 @@ class CatalogProperties:
         root: Optional[str] = None,
         filesystem: Optional[pyarrow.fs.FileSystem] = None,
         storage=None,
-        *args,
-        **kwargs,
     ):
         """
         Initialize a CatalogProperties instance.
 
         Args:
-            root: A single directory path that serves as the catalog root dir.
+            root: Catalog root directory path. Uses the "DELTACAT_ROOT"
+                system environment variable if not set, and defaults to
+                "./.deltacat/" if this environment variable is not set.
             filesystem: The filesystem implementation that should be used for
                 reading these files. If None, a filesystem will be inferred.
-                If not None, the provided filesystem will still be validated
-                against the provided path to ensure compatibility.
+                If provided, this will be validated for compatibility with the
+                catalog root path.
         """
         # set root, using precedence rules described in pydoc
         if root is None:
             # Check environment variables
-            # This is set or defaulted in constants.py
             root = DELTACAT_ROOT
-            if root is None:
-                raise ValueError(
-                    "Expected environment variable DELTACAT_ROOT to be set or defaulted"
-                )
+            if not root:
+                # Default to "./.deltacat/"
+                root = os.path.join(os.getcwd(), ".deltacat")
 
         resolved_root, resolved_filesystem = resolve_path_and_filesystem(
             path=root,

deltacat/compute/__init__.py

@@ -0,0 +1,14 @@
+from deltacat.compute.jobs.client import (
+    DeltaCatJobClient,
+    job_client,
+    local_job_client,
+)
+
+from ray.job_submission import JobStatus
+
+__all__ = [
+    "job_client",
+    "local_job_client",
+    "DeltaCatJobClient",
+    "JobStatus",
+]

deltacat/compute/converter/constants.py

@@ -2,3 +2,8 @@ DEFAULT_CONVERTER_TASK_MAX_PARALLELISM = 4096
 
 # Safe limit ONLY considering CPU limit, typically 32 for a 8x-large worker
 DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD = 30
+
+
+# Unique identifier delimiter to ensure different primary key don't end up with same hash when concatenated.
+# e.g.: pk column a with value: 1, 12; pk column b with value: 12, 1; Without delimiter will both become "121".
+IDENTIFIER_FIELD_DELIMITER = "c303282d"

deltacat/compute/converter/converter_session.py

@@ -1,4 +1,3 @@
-# from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties
 from deltacat.utils.ray_utils.concurrency import (
     invoke_parallel,
     task_resource_options_provider,
@@ -20,7 +19,6 @@ from deltacat.compute.converter.steps.convert import convert
 from deltacat.compute.converter.model.convert_input import ConvertInput
 from deltacat.compute.converter.pyiceberg.overrides import (
     fetch_all_bucket_files,
-    parquet_files_dict_to_iceberg_data_files,
 )
 from deltacat.compute.converter.utils.converter_session_utils import (
     construct_iceberg_table_prefix,
@@ -48,32 +46,46 @@ def converter_session(params: ConverterSessionParams, **kwargs):
     table_name = params.iceberg_table_name
     iceberg_table = load_table(catalog, table_name)
     enforce_primary_key_uniqueness = params.enforce_primary_key_uniqueness
+    iceberg_warehouse_bucket_name = params.iceberg_warehouse_bucket_name
+    iceberg_namespace = params.iceberg_namespace
+    merge_keys = params.merge_keys
+    compact_previous_position_delete_files = (
+        params.compact_previous_position_delete_files
+    )
+    task_max_parallelism = params.task_max_parallelism
+    s3_client_kwargs = params.s3_client_kwargs
+    s3_file_system = params.s3_file_system
+    location_provider_prefix_override = params.location_provider_prefix_override
+    position_delete_for_multiple_data_files = (
+        params.position_delete_for_multiple_data_files
+    )
+
     data_file_dict, equality_delete_dict, pos_delete_dict = fetch_all_bucket_files(
         iceberg_table
     )
+
     convert_input_files_for_all_buckets = group_all_files_to_each_bucket(
         data_file_dict=data_file_dict,
         equality_delete_dict=equality_delete_dict,
         pos_delete_dict=pos_delete_dict,
     )
-    iceberg_warehouse_bucket_name = params.iceberg_warehouse_bucket_name
-    iceberg_namespace = params.iceberg_namespace
-    iceberg_table_warehouse_prefix = construct_iceberg_table_prefix(
-        iceberg_warehouse_bucket_name=iceberg_warehouse_bucket_name,
-        table_name=table_name,
-        iceberg_namespace=iceberg_namespace,
-    )
-    merge_keys = params.merge_keys
+
+    if not location_provider_prefix_override:
+        iceberg_table_warehouse_prefix = construct_iceberg_table_prefix(
+            iceberg_warehouse_bucket_name=iceberg_warehouse_bucket_name,
+            table_name=table_name,
+            iceberg_namespace=iceberg_namespace,
+        )
+    else:
+        iceberg_table_warehouse_prefix = location_provider_prefix_override
+
     # Using table identifier fields as merge keys if merge keys not provided
     if not merge_keys:
         identifier_fields_set = iceberg_table.schema().identifier_field_names()
         identifier_fields = list(identifier_fields_set)
     else:
         identifier_fields = merge_keys
-    if len(identifier_fields) > 1:
-        raise NotImplementedError(
-            f"Multiple identifier fields lookup not supported yet."
-        )
+
     convert_options_provider = functools.partial(
         task_resource_options_provider,
         resource_amount_provider=convert_resource_options_provider,
@@ -86,58 +98,88 @@ def converter_session(params: ConverterSessionParams, **kwargs):
     #  Note that approach 2 will ideally require shared object store to avoid download equality delete files * number of child tasks times.
     max_parallel_data_file_download = DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD
 
-    compact_small_files = params.compact_small_files
-    position_delete_for_multiple_data_files = (
-        params.position_delete_for_multiple_data_files
-    )
-    task_max_parallelism = params.task_max_parallelism
-
     def convert_input_provider(index, item):
         return {
             "convert_input": ConvertInput.of(
-                files_for_each_bucket=item,
+                convert_input_files=item,
                 convert_task_index=index,
                 iceberg_table_warehouse_prefix=iceberg_table_warehouse_prefix,
                 identifier_fields=identifier_fields,
-                compact_small_files=compact_small_files,
+                compact_previous_position_delete_files=compact_previous_position_delete_files,
+                table_io=iceberg_table.io,
+                table_metadata=iceberg_table.metadata,
                 enforce_primary_key_uniqueness=enforce_primary_key_uniqueness,
                 position_delete_for_multiple_data_files=position_delete_for_multiple_data_files,
                 max_parallel_data_file_download=max_parallel_data_file_download,
+                s3_client_kwargs=s3_client_kwargs,
+                s3_file_system=s3_file_system,
             )
         }
 
+    logger.info(f"Getting remote convert tasks...")
     # Ray remote task: convert
-    # Assuming that memory consume by each bucket doesn't exceed one node's memory limit.
     # TODO: Add split mechanism to split large buckets
     convert_tasks_pending = invoke_parallel(
-        items=convert_input_files_for_all_buckets.items(),
+        items=convert_input_files_for_all_buckets,
         ray_task=convert,
         max_parallelism=task_max_parallelism,
         options_provider=convert_options_provider,
         kwargs_provider=convert_input_provider,
     )
+
     to_be_deleted_files_list = []
-    to_be_added_files_dict_list = []
+    logger.info(f"Finished invoking {len(convert_tasks_pending)} convert tasks.")
+
     convert_results = ray.get(convert_tasks_pending)
-    for convert_result in convert_results:
-        to_be_deleted_files_list.extend(convert_result[0].values())
-        to_be_added_files_dict_list.append(convert_result[1])
+    logger.info(f"Got {len(convert_tasks_pending)} convert tasks.")
 
-    new_position_delete_files = parquet_files_dict_to_iceberg_data_files(
-        io=iceberg_table.io,
-        table_metadata=iceberg_table.metadata,
-        files_dict_list=to_be_added_files_dict_list,
+    total_position_delete_record_count = sum(
+        convert_result.position_delete_record_count
+        for convert_result in convert_results
+    )
+    total_input_data_file_record_count = sum(
+        convert_result.input_data_files_record_count
+        for convert_result in convert_results
+    )
+    total_data_file_hash_columns_in_memory_sizes = sum(
+        convert_result.input_data_files_hash_columns_in_memory_sizes
+        for convert_result in convert_results
+    )
+    total_position_delete_file_in_memory_sizes = sum(
+        convert_result.position_delete_in_memory_sizes
+        for convert_result in convert_results
+    )
+    total_position_delete_on_disk_sizes = sum(
+        convert_result.position_delete_on_disk_sizes
+        for convert_result in convert_results
     )
 
-    if not to_be_deleted_files_list:
+    to_be_added_files_list = []
+    for convert_result in convert_results:
+        to_be_added_files = convert_result.to_be_added_files
+        to_be_deleted_files = convert_result.to_be_deleted_files
+
+        to_be_deleted_files_list.extend(to_be_deleted_files.values())
+        to_be_added_files_list.extend(to_be_added_files)
+
+    if not to_be_deleted_files_list and to_be_added_files_list:
         commit_append_snapshot(
             iceberg_table=iceberg_table,
-            new_position_delete_files=new_position_delete_files,
+            new_position_delete_files=to_be_added_files_list,
         )
     else:
         commit_replace_snapshot(
             iceberg_table=iceberg_table,
-            # equality_delete_files + data file that all rows are deleted
             to_be_deleted_files_list=to_be_deleted_files_list,
-            new_position_delete_files=new_position_delete_files,
+            new_position_delete_files=to_be_added_files_list,
         )
+    logger.info(
+        f"Aggregated stats for {table_name}: "
+        f"total position delete record count: {total_position_delete_record_count}, "
+        f"total input data file record_count: {total_input_data_file_record_count}, "
+        f"total data file hash columns in memory sizes: {total_data_file_hash_columns_in_memory_sizes}, "
+        f"total position delete file in memory sizes: {total_position_delete_file_in_memory_sizes}, "
+        f"total position delete file on disk sizes: {total_position_delete_on_disk_sizes}."
+    )
+
+    logger.info(f"Committed new Iceberg snapshot.")

deltacat/compute/converter/model/convert_input.py

@@ -10,11 +10,14 @@ class ConvertInput(Dict):
         convert_task_index,
         iceberg_table_warehouse_prefix,
         identifier_fields,
-        compact_small_files,
+        table_io,
+        table_metadata,
+        compact_previous_position_delete_files,
         enforce_primary_key_uniqueness,
         position_delete_for_multiple_data_files,
         max_parallel_data_file_download,
         s3_file_system,
+        s3_client_kwargs,
     ) -> ConvertInput:
 
         result = ConvertInput()
@@ -22,13 +25,18 @@ class ConvertInput(Dict):
         result["convert_task_index"] = convert_task_index
         result["identifier_fields"] = identifier_fields
         result["iceberg_table_warehouse_prefix"] = iceberg_table_warehouse_prefix
-        result["compact_small_files"] = compact_small_files
+        result["table_io"] = table_io
+        result["table_metadata"] = table_metadata
+        result[
+            "compact_previous_position_delete_files"
+        ] = compact_previous_position_delete_files
         result["enforce_primary_key_uniqueness"] = enforce_primary_key_uniqueness
         result[
             "position_delete_for_multiple_data_files"
         ] = position_delete_for_multiple_data_files
         result["max_parallel_data_file_download"] = max_parallel_data_file_download
         result["s3_file_system"] = s3_file_system
+        result["s3_client_kwargs"] = s3_client_kwargs
 
         return result
 
@@ -49,8 +57,16 @@ class ConvertInput(Dict):
         return self["iceberg_table_warehouse_prefix"]
 
     @property
-    def compact_small_files(self) -> bool:
-        return self["compact_small_files"]
+    def table_io(self):
+        return self["table_io"]
+
+    @property
+    def table_metadata(self):
+        return self["table_metadata"]
+
+    @property
+    def compact_previous_position_delete_files(self) -> bool:
+        return self["compact_previous_position_delete_files"]
 
     @property
     def enforce_primary_key_uniqueness(self) -> bool:
@@ -67,3 +83,7 @@ class ConvertInput(Dict):
     @property
     def s3_file_system(self):
         return self["s3_file_system"]
+
+    @property
+    def s3_client_kwargs(self):
+        return self["s3_client_kwargs"]

deltacat/compute/converter/model/convert_result.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+from typing import Dict
+
+
+class ConvertResult(Dict):
+    @staticmethod
+    def of(
+        convert_task_index,
+        to_be_added_files,
+        to_be_deleted_files,
+        position_delete_record_count,
+        input_data_files_record_count,
+        input_data_files_hash_columns_in_memory_sizes,
+        position_delete_in_memory_sizes,
+        position_delete_on_disk_sizes,
+    ) -> ConvertResult:
+
+        result = ConvertResult()
+        result["convert_task_index"] = convert_task_index
+        result["to_be_added_files"] = to_be_added_files
+        result["to_be_deleted_files"] = to_be_deleted_files
+        result["position_delete_record_count"] = position_delete_record_count
+        result["input_data_files_record_count"] = input_data_files_record_count
+        result[
+            "input_data_files_hash_columns_in_memory_sizes"
+        ] = input_data_files_hash_columns_in_memory_sizes
+        result["position_delete_in_memory_sizes"] = position_delete_in_memory_sizes
+        result["position_delete_on_disk_sizes"] = position_delete_on_disk_sizes
+        return result
+
+    @property
+    def convert_task_index(self) -> int:
+        return self["convert_task_index"]
+
+    @property
+    def to_be_added_files(self):
+        return self["to_be_added_files"]
+
+    @property
+    def to_be_deleted_files(self):
+        return self["to_be_deleted_files"]
+
+    @property
+    def position_delete_record_count(self):
+        return self["position_delete_record_count"]
+
+    @property
+    def input_data_files_record_count(self):
+        return self["input_data_files_record_count"]
+
+    @property
+    def input_data_files_hash_columns_in_memory_sizes(self):
+        return self["input_data_files_hash_columns_in_memory_sizes"]
+
+    @property
+    def position_delete_in_memory_sizes(self):
+        return self["position_delete_in_memory_sizes"]
+
+    @property
+    def position_delete_on_disk_sizes(self):
+        return self["position_delete_on_disk_sizes"]

deltacat/compute/converter/model/converter_session_params.py

@@ -1,6 +1,10 @@
 from __future__ import annotations
 from typing import Optional, Dict
-from deltacat.compute.converter.constants import DEFAULT_CONVERTER_TASK_MAX_PARALLELISM
+from deltacat.compute.converter.constants import (
+    DEFAULT_CONVERTER_TASK_MAX_PARALLELISM,
+)
+from deltacat.constants import DEFAULT_NAMESPACE
+from fsspec import AbstractFileSystem
 
 
 class ConverterSessionParams(dict):
@@ -18,15 +22,15 @@ class ConverterSessionParams(dict):
         assert (
             params.get("iceberg_warehouse_bucket_name") is not None
         ), "iceberg_warehouse_bucket_name is a required arg"
-        assert (
-            params.get("iceberg_namespace") is not None
-        ), "iceberg_namespace is a required arg"
         result = ConverterSessionParams(params)
 
+        result.iceberg_namespace = params.get("iceberg_namespace", DEFAULT_NAMESPACE)
         result.enforce_primary_key_uniqueness = params.get(
             "enforce_primary_key_uniqueness", False
         )
-        result.compact_small_files = params.get("compact_small_files", False)
+        result.compact_previous_position_delete_files = params.get(
+            "compact_previous_position_delete_files", False
+        )
 
         # For Iceberg v3 spec, option to produce delete vector that can establish 1:1 mapping with data files.
         result.position_delete_for_multiple_data_files = params.get(
@@ -36,6 +40,10 @@ class ConverterSessionParams(dict):
             "task_max_parallelism", DEFAULT_CONVERTER_TASK_MAX_PARALLELISM
         )
         result.merge_keys = params.get("merge_keys", None)
+        result.s3_client_kwargs = params.get("s3_client_kwargs", {})
+        result.s3_file_system = params.get("s3_file_system", None)
+        result.s3_prefix_override = params.get("s3_prefix_override", None)
+
         return result
 
     @property
@@ -54,6 +62,10 @@ class ConverterSessionParams(dict):
     def iceberg_namespace(self) -> str:
         return self["iceberg_namespace"]
 
+    @iceberg_namespace.setter
+    def iceberg_namespace(self, iceberg_namespace) -> None:
+        self["iceberg_namespace"] = iceberg_namespace
+
     @property
     def enforce_primary_key_uniqueness(self) -> bool:
         return self["enforce_primary_key_uniqueness"]
@@ -63,12 +75,16 @@ class ConverterSessionParams(dict):
         self["enforce_primary_key_uniqueness"] = enforce_primary_key_uniqueness
 
     @property
-    def compact_small_files(self) -> bool:
-        return self["compact_small_files"]
+    def compact_previous_position_delete_files(self) -> bool:
+        return self["compact_previous_position_delete_files"]
 
-    @compact_small_files.setter
-    def compact_small_files(self, compact_small_files) -> None:
-        self["compact_small_files"] = compact_small_files
+    @compact_previous_position_delete_files.setter
+    def compact_previous_position_delete_files(
+        self, compact_previous_position_delete_files
+    ) -> None:
+        self[
+            "compact_previous_position_delete_files"
+        ] = compact_previous_position_delete_files
 
     @property
     def position_delete_for_multiple_data_files(self) -> bool:
@@ -97,3 +113,29 @@ class ConverterSessionParams(dict):
     @merge_keys.setter
     def merge_keys(self, merge_keys) -> None:
         self["merge_keys"] = merge_keys
+
+    @property
+    def s3_client_kwargs(self) -> Dict:
+        return self["s3_client_kwargs"]
+
+    @s3_client_kwargs.setter
+    def s3_client_kwargs(self, s3_client_kwargs) -> None:
+        self["s3_client_kwargs"] = s3_client_kwargs
+
+    @property
+    def s3_file_system(self) -> AbstractFileSystem:
+        return self["s3_file_system"]
+
+    @s3_file_system.setter
+    def s3_file_system(self, s3_file_system) -> None:
+        self["s3_file_system"] = s3_file_system
+
+    @property
+    def location_provider_prefix_override(self) -> str:
+        return self["location_provider_prefix_override"]
+
+    @location_provider_prefix_override.setter
+    def location_provider_prefix_override(
+        self, location_provider_prefix_override
+    ) -> None:
+        self["location_provider_prefix_override"] = location_provider_prefix_override