deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/compute/converter/model/convert_input.py

@@ -1,23 +1,25 @@
 from __future__ import annotations
-from typing import Dict, List
+from typing import Dict, List, Any, Optional
 from deltacat.compute.converter.model.convert_input_files import ConvertInputFiles
+from fsspec import AbstractFileSystem
 
 
 class ConvertInput(Dict):
     @staticmethod
     def of(
-        convert_input_files,
-        convert_task_index,
-        iceberg_table_warehouse_prefix,
-        identifier_fields,
-        table_io,
-        table_metadata,
-        compact_previous_position_delete_files,
-        enforce_primary_key_uniqueness,
-        position_delete_for_multiple_data_files,
-        max_parallel_data_file_download,
-        s3_file_system,
-        s3_client_kwargs,
+        convert_input_files: ConvertInputFiles,
+        convert_task_index: int,
+        iceberg_table_warehouse_prefix: str,
+        identifier_fields: List[str],
+        table_io: Any,
+        table_metadata: Any,
+        compact_previous_position_delete_files: bool,
+        enforce_primary_key_uniqueness: bool,
+        position_delete_for_multiple_data_files: bool,
+        max_parallel_data_file_download: int,
+        filesystem: Optional[AbstractFileSystem],
+        s3_client_kwargs: Optional[Dict[str, Any]],
+        task_memory: float,
     ) -> ConvertInput:
 
         result = ConvertInput()
@@ -35,8 +37,9 @@ class ConvertInput(Dict):
             "position_delete_for_multiple_data_files"
         ] = position_delete_for_multiple_data_files
         result["max_parallel_data_file_download"] = max_parallel_data_file_download
-        result["s3_file_system"] = s3_file_system
+        result["filesystem"] = filesystem
         result["s3_client_kwargs"] = s3_client_kwargs
+        result["task_memory"] = task_memory
 
         return result
 
@@ -57,11 +60,11 @@ class ConvertInput(Dict):
         return self["iceberg_table_warehouse_prefix"]
 
     @property
-    def table_io(self):
+    def table_io(self) -> Any:
         return self["table_io"]
 
     @property
-    def table_metadata(self):
+    def table_metadata(self) -> Any:
         return self["table_metadata"]
 
     @property
@@ -81,9 +84,13 @@ class ConvertInput(Dict):
         return self["max_parallel_data_file_download"]
 
     @property
-    def s3_file_system(self):
-        return self["s3_file_system"]
+    def filesystem(self) -> Optional[AbstractFileSystem]:
+        return self["filesystem"]
 
     @property
-    def s3_client_kwargs(self):
+    def s3_client_kwargs(self) -> Optional[Dict[str, Any]]:
         return self["s3_client_kwargs"]
+
+    @property
+    def task_memory(self) -> float:
+        return self["task_memory"]

deltacat/compute/converter/model/convert_input_files.py

@@ -1,15 +1,21 @@
 from __future__ import annotations
-from typing import Dict
+from typing import Dict, List, Any, Optional, Tuple
+from pyiceberg.manifest import DataFile
+
+# Type aliases to simplify nested types
+DataFileWithSequence = Tuple[int, DataFile]  # (sequence_number, data_file)
+DataFileList = List[DataFileWithSequence]  # List of data files with sequence numbers
+DataFileListGroup = List[DataFileList]  # Group of data file lists
 
 
 class ConvertInputFiles(Dict):
     @staticmethod
     def of(
-        partition_value,
-        all_data_files_for_dedupe=None,
-        applicable_data_files=None,
-        applicable_equality_delete_files=None,
-        existing_position_delete_files=None,
+        partition_value: Any,
+        all_data_files_for_dedupe: Optional[DataFileList] = None,
+        applicable_data_files: Optional[DataFileListGroup] = None,
+        applicable_equality_delete_files: Optional[DataFileListGroup] = None,
+        existing_position_delete_files: Optional[DataFileList] = None,
     ) -> ConvertInputFiles:
 
         result = ConvertInputFiles()
@@ -21,41 +27,52 @@ class ConvertInputFiles(Dict):
         return result
 
     @property
-    def partition_value(self):
+    def partition_value(self) -> Any:
         return self["partition_value"]
 
     @property
-    def all_data_files_for_dedupe(self):
+    def all_data_files_for_dedupe(self) -> Optional[DataFileList]:
         return self["all_data_files_for_dedupe"]
 
     @property
-    def applicable_data_files(self):
+    def applicable_data_files(self) -> Optional[DataFileListGroup]:
         return self["applicable_data_files"]
 
     @property
-    def applicable_equality_delete_files(self):
+    def applicable_equality_delete_files(
+        self,
+    ) -> Optional[DataFileListGroup]:
         return self["applicable_equality_delete_files"]
 
     @property
-    def existing_position_delete_files(self):
+    def existing_position_delete_files(self) -> Optional[DataFileList]:
         return self["existing_position_delete_files"]
 
     @partition_value.setter
-    def partition_value(self, partition_value):
+    def partition_value(self, partition_value: Any) -> None:
         self["partition_value"] = partition_value
 
     @all_data_files_for_dedupe.setter
-    def all_data_files_for_dedupe(self, all_data_files_for_dedupe):
+    def all_data_files_for_dedupe(
+        self, all_data_files_for_dedupe: Optional[DataFileList]
+    ) -> None:
         self["all_data_files_for_dedupe"] = all_data_files_for_dedupe
 
     @applicable_data_files.setter
-    def applicable_data_files(self, applicable_data_files):
+    def applicable_data_files(
+        self, applicable_data_files: Optional[DataFileListGroup]
+    ) -> None:
         self["applicable_data_files"] = applicable_data_files
 
     @applicable_equality_delete_files.setter
-    def applicable_equality_delete_files(self, applicable_equality_delete_files):
+    def applicable_equality_delete_files(
+        self,
+        applicable_equality_delete_files: Optional[DataFileListGroup],
+    ) -> None:
         self["applicable_equality_delete_files"] = applicable_equality_delete_files
 
     @existing_position_delete_files.setter
-    def existing_position_delete_files(self, existing_position_delete_files):
+    def existing_position_delete_files(
+        self, existing_position_delete_files: Optional[DataFileList]
+    ) -> None:
         self["existing_position_delete_files"] = existing_position_delete_files

deltacat/compute/converter/model/convert_result.py

@@ -1,18 +1,22 @@
 from __future__ import annotations
-from typing import Dict
+from typing import Dict, List, Any
+from pyiceberg.manifest import DataFile
 
 
 class ConvertResult(Dict):
     @staticmethod
     def of(
-        convert_task_index,
-        to_be_added_files,
-        to_be_deleted_files,
-        position_delete_record_count,
-        input_data_files_record_count,
-        input_data_files_hash_columns_in_memory_sizes,
-        position_delete_in_memory_sizes,
-        position_delete_on_disk_sizes,
+        convert_task_index: int,
+        to_be_added_files: List[DataFile],
+        to_be_deleted_files: Dict[Any, List[DataFile]],
+        position_delete_record_count: int,
+        input_data_files_record_count: int,
+        input_data_files_hash_columns_in_memory_sizes: int,
+        position_delete_in_memory_sizes: int,
+        position_delete_on_disk_sizes: int,
+        input_data_files_on_disk_size: int,
+        peak_memory_usage_bytes: int,
+        memory_usage_percentage: float,
     ) -> ConvertResult:
 
         result = ConvertResult()
@@ -26,6 +30,9 @@ class ConvertResult(Dict):
         ] = input_data_files_hash_columns_in_memory_sizes
         result["position_delete_in_memory_sizes"] = position_delete_in_memory_sizes
         result["position_delete_on_disk_sizes"] = position_delete_on_disk_sizes
+        result["input_data_files_on_disk_size"] = input_data_files_on_disk_size
+        result["peak_memory_usage_bytes"] = peak_memory_usage_bytes
+        result["memory_usage_percentage"] = memory_usage_percentage
         return result
 
     @property
@@ -33,29 +40,41 @@ class ConvertResult(Dict):
         return self["convert_task_index"]
 
     @property
-    def to_be_added_files(self):
+    def to_be_added_files(self) -> List[DataFile]:
         return self["to_be_added_files"]
 
     @property
-    def to_be_deleted_files(self):
+    def to_be_deleted_files(self) -> Dict[Any, List[DataFile]]:
         return self["to_be_deleted_files"]
 
     @property
-    def position_delete_record_count(self):
+    def position_delete_record_count(self) -> int:
         return self["position_delete_record_count"]
 
     @property
-    def input_data_files_record_count(self):
+    def input_data_files_record_count(self) -> int:
         return self["input_data_files_record_count"]
 
     @property
-    def input_data_files_hash_columns_in_memory_sizes(self):
+    def input_data_files_hash_columns_in_memory_sizes(self) -> int:
         return self["input_data_files_hash_columns_in_memory_sizes"]
 
     @property
-    def position_delete_in_memory_sizes(self):
+    def position_delete_in_memory_sizes(self) -> int:
         return self["position_delete_in_memory_sizes"]
 
     @property
-    def position_delete_on_disk_sizes(self):
+    def position_delete_on_disk_sizes(self) -> int:
         return self["position_delete_on_disk_sizes"]
+
+    @property
+    def input_data_files_on_disk_size(self) -> int:
+        return self["input_data_files_on_disk_size"]
+
+    @property
+    def peak_memory_usage_bytes(self) -> int:
+        return self["peak_memory_usage_bytes"]
+
+    @property
+    def memory_usage_percentage(self) -> float:
+        return self["memory_usage_percentage"]

deltacat/compute/converter/model/converter_session_params.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
-from typing import Optional, Dict
+from typing import Optional, Dict, Any, List
 from deltacat.compute.converter.constants import (
     DEFAULT_CONVERTER_TASK_MAX_PARALLELISM,
 )
 from deltacat.constants import DEFAULT_NAMESPACE
 from fsspec import AbstractFileSystem
+from pyiceberg.catalog import Catalog
 
 
 class ConverterSessionParams(dict):
@@ -13,7 +14,7 @@ class ConverterSessionParams(dict):
     """
 
     @staticmethod
-    def of(params: Optional[Dict]) -> ConverterSessionParams:
+    def of(params: Optional[Dict[str, Any]]) -> ConverterSessionParams:
         params = {} if params is None else params
         assert params.get("catalog") is not None, "catalog is a required arg"
         assert (
@@ -41,13 +42,13 @@ class ConverterSessionParams(dict):
         )
         result.merge_keys = params.get("merge_keys", None)
         result.s3_client_kwargs = params.get("s3_client_kwargs", {})
-        result.s3_file_system = params.get("s3_file_system", None)
+        result.filesystem = params.get("filesystem", None)
         result.s3_prefix_override = params.get("s3_prefix_override", None)
 
         return result
 
     @property
-    def catalog(self):
+    def catalog(self) -> Catalog:
         return self["catalog"]
 
     @property
@@ -63,7 +64,7 @@ class ConverterSessionParams(dict):
         return self["iceberg_namespace"]
 
     @iceberg_namespace.setter
-    def iceberg_namespace(self, iceberg_namespace) -> None:
+    def iceberg_namespace(self, iceberg_namespace: str) -> None:
         self["iceberg_namespace"] = iceberg_namespace
 
     @property
@@ -71,7 +72,9 @@ class ConverterSessionParams(dict):
         return self["enforce_primary_key_uniqueness"]
 
     @enforce_primary_key_uniqueness.setter
-    def enforce_primary_key_uniqueness(self, enforce_primary_key_uniqueness) -> None:
+    def enforce_primary_key_uniqueness(
+        self, enforce_primary_key_uniqueness: bool
+    ) -> None:
         self["enforce_primary_key_uniqueness"] = enforce_primary_key_uniqueness
 
     @property
@@ -80,7 +83,7 @@ class ConverterSessionParams(dict):
 
     @compact_previous_position_delete_files.setter
     def compact_previous_position_delete_files(
-        self, compact_previous_position_delete_files
+        self, compact_previous_position_delete_files: bool
     ) -> None:
         self[
             "compact_previous_position_delete_files"
@@ -92,50 +95,50 @@ class ConverterSessionParams(dict):
 
     @position_delete_for_multiple_data_files.setter
     def position_delete_for_multiple_data_files(
-        self, position_delete_for_multiple_data_files
+        self, position_delete_for_multiple_data_files: bool
     ) -> None:
         self[
             "position_delete_for_multiple_data_files"
         ] = position_delete_for_multiple_data_files
 
     @property
-    def task_max_parallelism(self) -> str:
+    def task_max_parallelism(self) -> int:
         return self["task_max_parallelism"]
 
     @task_max_parallelism.setter
-    def task_max_parallelism(self, task_max_parallelism) -> None:
+    def task_max_parallelism(self, task_max_parallelism: int) -> None:
         self["task_max_parallelism"] = task_max_parallelism
 
     @property
-    def merge_keys(self) -> str:
+    def merge_keys(self) -> Optional[List[str]]:
         return self["merge_keys"]
 
     @merge_keys.setter
-    def merge_keys(self, merge_keys) -> None:
+    def merge_keys(self, merge_keys: Optional[List[str]]) -> None:
         self["merge_keys"] = merge_keys
 
     @property
-    def s3_client_kwargs(self) -> Dict:
+    def s3_client_kwargs(self) -> Dict[str, Any]:
         return self["s3_client_kwargs"]
 
     @s3_client_kwargs.setter
-    def s3_client_kwargs(self, s3_client_kwargs) -> None:
+    def s3_client_kwargs(self, s3_client_kwargs: Dict[str, Any]) -> None:
         self["s3_client_kwargs"] = s3_client_kwargs
 
     @property
-    def s3_file_system(self) -> AbstractFileSystem:
-        return self["s3_file_system"]
+    def filesystem(self) -> Optional[AbstractFileSystem]:
+        return self["filesystem"]
 
-    @s3_file_system.setter
-    def s3_file_system(self, s3_file_system) -> None:
-        self["s3_file_system"] = s3_file_system
+    @filesystem.setter
+    def filesystem(self, filesystem: Optional[AbstractFileSystem]) -> None:
+        self["filesystem"] = filesystem
 
     @property
-    def location_provider_prefix_override(self) -> str:
+    def location_provider_prefix_override(self) -> Optional[str]:
         return self["location_provider_prefix_override"]
 
     @location_provider_prefix_override.setter
     def location_provider_prefix_override(
-        self, location_provider_prefix_override
+        self, location_provider_prefix_override: Optional[str]
     ) -> None:
         self["location_provider_prefix_override"] = location_provider_prefix_override

deltacat/compute/converter/pyiceberg/catalog.py

@@ -1,8 +1,15 @@
-from typing import Optional
-
-
-def load_catalog(iceberg_catalog_name, iceberg_catalog_properties):
-    catalog = load_catalog(
+from typing import Optional, Dict, Any
+from pyiceberg.table import Table
+from pyiceberg.catalog import Catalog, load_catalog as pyiceberg_load_catalog
+from botocore.credentials import Credentials
+import boto3
+from boto3.session import Session
+
+
+def load_catalog(
+    iceberg_catalog_name: str, iceberg_catalog_properties: Dict[str, Any]
+) -> Catalog:
+    catalog = pyiceberg_load_catalog(
         name=iceberg_catalog_name,
         **iceberg_catalog_properties,
     )
@@ -23,25 +30,21 @@ def get_s3_path(
     return result_path
 
 
-def get_bucket_name():
-    return "metadata-py4j-zyiqin1"
+def get_bucket_name() -> str:
+    return "test-bucket"
 
 
-def get_s3_prefix():
+def get_s3_prefix() -> str:
     return get_s3_path(get_bucket_name())
 
 
-def get_credential():
-    import boto3
-
-    boto3_session = boto3.Session()
-    credentials = boto3_session.get_credentials()
+def get_credential() -> Credentials:
+    boto3_session: Session = boto3.Session()
+    credentials: Credentials = boto3_session.get_credentials()
     return credentials
 
 
-def get_glue_catalog():
-    from pyiceberg.catalog import load_catalog
-
+def get_glue_catalog() -> Catalog:
     credential = get_credential()
     # Credentials are refreshable, so accessing your access key / secret key
     # separately can lead to a race condition. Use this to get an actual matched
@@ -51,7 +54,7 @@ def get_glue_catalog():
     secret_access_key = credential.secret_key
     session_token = credential.token
     s3_path = get_s3_prefix()
-    glue_catalog = load_catalog(
+    glue_catalog = pyiceberg_load_catalog(
         "glue",
         **{
             "warehouse": s3_path,
@@ -70,6 +73,6 @@ def get_glue_catalog():
     return glue_catalog
 
 
-def load_table(catalog, table_name):
+def load_table(catalog: Catalog, table_name: str) -> Table:
     loaded_table = catalog.load_table(table_name)
     return loaded_table

deltacat/compute/converter/pyiceberg/overrides.py

@@ -11,7 +11,7 @@ from pyiceberg.io.pyarrow import (
     MetricsMode,
     StatsAggregator,
 )
-from typing import Dict, List, Set
+from typing import Dict, List, Set, Any, Tuple
 from deltacat.compute.converter.utils.iceberg_columns import (
     ICEBERG_RESERVED_FIELD_ID_FOR_FILE_PATH_COLUMN,
     ICEBERG_RESERVED_FIELD_ID_FOR_POS_COLUMN,
@@ -24,18 +24,23 @@ from pyiceberg.manifest import (
     DataFileContent,
     FileFormat,
 )
-from pyiceberg.table import _min_sequence_number, _open_manifest
+from pyiceberg.table import _min_sequence_number, _open_manifest, Table
 from pyiceberg.utils.concurrent import ExecutorFactory
 from itertools import chain
 from pyiceberg.typedef import (
     KeyDefaultDict,
 )
+from pyiceberg.schema import Schema
+from pyiceberg.io import FileIO
+from deltacat.compute.converter.model.convert_input_files import (
+    DataFileList,
+)
 
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-def parquet_path_to_id_mapping_override(schema):
+def parquet_path_to_id_mapping_override(schema: Schema) -> Dict[str, int]:
     res = parquet_path_to_id_mapping(schema)
     # Override here to insert position delete reserved column field IDs
     res["file_path"] = ICEBERG_RESERVED_FIELD_ID_FOR_FILE_PATH_COLUMN
@@ -155,13 +160,16 @@ def data_file_statistics_from_parquet_metadata(
     )
 
 
-def parquet_files_dict_to_iceberg_data_files(io, table_metadata, files_dict):
-    data_file_content_type = DataFileContent.POSITION_DELETES
+def parquet_files_dict_to_iceberg_data_files(
+    io: FileIO,
+    table_metadata: Any,
+    files_dict: Dict[Any, List[str]],
+    file_content_type: DataFileContent,
+) -> List[DataFile]:
     iceberg_files = []
     schema = table_metadata.schema()
     for partition_value, file_paths in files_dict.items():
         for file_path in file_paths:
-            logger.info(f"DEBUG_file_path:{file_path}")
             input_file = io.new_input(file_path)
             with input_file.open() as input_stream:
                 parquet_metadata = pq.read_metadata(input_stream)
@@ -177,7 +185,7 @@ def parquet_files_dict_to_iceberg_data_files(io, table_metadata, files_dict):
             )
 
             data_file = DataFile(
-                content=data_file_content_type,
+                content=file_content_type,
                 file_path=file_path,
                 file_format=FileFormat.PARQUET,
                 partition=partition_value,
@@ -192,10 +200,11 @@ def parquet_files_dict_to_iceberg_data_files(io, table_metadata, files_dict):
     return iceberg_files
 
 
-def fetch_all_bucket_files(table):
+def fetch_all_bucket_files(
+    table: Table,
+) -> Tuple[Dict[Any, DataFileList], Dict[Any, DataFileList], Dict[Any, DataFileList]]:
     # step 1: filter manifests using partition summaries
     # the filter depends on the partition spec used to write the manifest file, so create a cache of filters for each spec id
-
     data_scan = table.scan()
     snapshot = data_scan.snapshot()
     if not snapshot: