deltacat 0.2.10__py3-none-any.whl → 1.0.0__py3-none-any.whl

deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -32,6 +32,7 @@ from deltacat.tests.compute.compact_partition_test_cases import (
     REBASE_THEN_INCREMENTAL_TEST_CASES,
 )
 from typing import Any, Callable, Dict, List, Optional, Set
+from deltacat.types.media import StorageType
 
 DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
     "db_file_path",
@@ -272,7 +273,9 @@ def test_compact_partition_rebase_then_incremental(
     compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
         setup_s3_resource, rcf_file_s3_uri
     )
-    tables = ds.download_delta(compacted_delta_locator, **ds_mock_kwargs)
+    tables = ds.download_delta(
+        compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
+    )
     actual_rebase_compacted_table = pa.concat_tables(tables)
     # if no primary key is specified then sort by sort_key for consistent assertion
     sorting_cols: List[Any] = (
@@ -341,7 +344,11 @@ def test_compact_partition_rebase_then_incremental(
         **compaction_audit_obj
     )
 
-    tables = ds.download_delta(compacted_delta_locator_incremental, **ds_mock_kwargs)
+    tables = ds.download_delta(
+        compacted_delta_locator_incremental,
+        storage_type=StorageType.LOCAL,
+        **ds_mock_kwargs,
+    )
     actual_compacted_table = pa.concat_tables(tables)
     expected_terminal_compact_partition_result = (
         expected_terminal_compact_partition_result.combine_chunks().sort_by(

deltacat/tests/local_deltacat_storage/__init__.py

@@ -1,10 +1,12 @@
 from typing import Any, Callable, Dict, List, Optional, Set, Union, Tuple
 
 import pyarrow as pa
+import daft
 import json
 import sqlite3
 from sqlite3 import Cursor, Connection
 import uuid
+import ray
 import io
 
 from deltacat.tests.test_utils.storage import create_empty_delta
@@ -38,7 +40,13 @@ from deltacat.storage import (
     ManifestEntry,
     ManifestEntryList,
 )
-from deltacat.types.media import ContentType, StorageType, TableType, ContentEncoding
+from deltacat.types.media import (
+    ContentType,
+    StorageType,
+    TableType,
+    ContentEncoding,
+    DistributedDatasetType,
+)
 from deltacat.utils.common import ReadKwargsProvider
 
 SQLITE_CUR_ARG = "sqlite3_cur"
@@ -337,9 +345,10 @@ def download_delta(
     columns: Optional[List[str]] = None,
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
     ray_options_provider: Callable[[int, Any], Dict[str, Any]] = None,
+    distributed_dataset_type: DistributedDatasetType = DistributedDatasetType.RAY_DATASET,
     *args,
     **kwargs,
-) -> Union[LocalDataset, DistributedDataset]:
+) -> Union[LocalDataset, DistributedDataset]:  # type: ignore
     result = []
     manifest = get_delta_manifest(delta_like, *args, **kwargs)
 
@@ -356,6 +365,14 @@ def download_delta(
             )
         )
 
+    if storage_type == StorageType.DISTRIBUTED:
+        if distributed_dataset_type is DistributedDatasetType.DAFT:
+            return daft.from_arrow(result)
+        elif distributed_dataset_type is DistributedDatasetType.RAY_DATASET:
+            return ray.data.from_arrow(result)
+        else:
+            raise ValueError(f"Dataset type {distributed_dataset_type} not supported!")
+
     return result
 
 

deltacat/tests/test_utils/pyarrow.py

@@ -1,14 +1,18 @@
-from typing import List
+from typing import List, Optional
 import pyarrow as pa
-from deltacat.storage import Delta, Partition
+from deltacat.storage import Delta, Partition, PartitionLocator
 import deltacat.tests.local_deltacat_storage as ds
 
 
 def create_delta_from_csv_file(
-    namespace: str, file_paths: List[str], *args, **kwargs
+    namespace: str,
+    file_paths: List[str],
+    table_name: Optional[str] = None,
+    *args,
+    **kwargs
 ) -> Delta:
     staged_partition = stage_partition_from_file_paths(
-        namespace, file_paths, *args, **kwargs
+        namespace, file_paths, *args, table_name=table_name, **kwargs
     )
 
     committed_delta = commit_delta_to_staged_partition(
@@ -19,10 +23,15 @@ def create_delta_from_csv_file(
 
 
 def stage_partition_from_file_paths(
-    namespace: str, file_paths: List[str], *args, **kwargs
+    namespace: str,
+    file_paths: List[str],
+    table_name: Optional[str] = None,
+    *args,
+    **kwargs
 ) -> Partition:
     ds.create_namespace(namespace, {}, **kwargs)
-    table_name = "-".join(file_paths).replace("/", "_")
+    if table_name is None:
+        table_name = "-".join(file_paths).replace("/", "_")
     ds.create_table_version(namespace, table_name, "1", **kwargs)
     stream = ds.get_stream(namespace, table_name, "1", **kwargs)
     staged_partition = ds.stage_partition(stream, [], **kwargs)
@@ -31,19 +40,29 @@ def stage_partition_from_file_paths(
 
 def commit_delta_to_staged_partition(
     staged_partition, file_paths: List[str], *args, **kwargs
+) -> Delta:
+    committed_delta = commit_delta_to_partition(
+        staged_partition, file_paths=file_paths, *args, **kwargs
+    )
+    ds.commit_partition(staged_partition, **kwargs)
+    return committed_delta
+
+
+def commit_delta_to_partition(
+    partition: Partition, file_paths: List[str], *args, **kwargs
 ) -> Delta:
     tables = []
 
+    if isinstance(partition, PartitionLocator):
+        partition = ds.get_partition(
+            partition.stream_locator, partition.partition_values, *args, **kwargs
+        )
+
     for file_path in file_paths:
         table = pa.csv.read_csv(file_path)
         tables.append(table)
-    deltas = []
 
-    for table in tables:
-        delta = ds.stage_delta(table, staged_partition, **kwargs)
-        deltas.append(delta)
+    table = pa.concat_tables(tables)
+    staged_delta = ds.stage_delta(table, partition, **kwargs)
 
-    merged_delta = Delta.merge_deltas(deltas=deltas)
-    committed_delta = ds.commit_delta(merged_delta, **kwargs)
-    ds.commit_partition(staged_partition, **kwargs)
-    return committed_delta
+    return ds.commit_delta(staged_delta, **kwargs)

deltacat/tests/utils/test_daft.py

@@ -1,6 +1,6 @@
 import unittest
 from deltacat.types.media import ContentEncoding, ContentType
-from deltacat.utils.daft import daft_s3_file_to_table
+from deltacat.utils.daft import daft_s3_file_to_table, s3_files_to_dataframe
 
 from deltacat.utils.pyarrow import ReadKwargsProviderPyArrowSchemaOverride
 from deltacat.types.partial_download import PartialParquetParameters
@@ -9,7 +9,7 @@ import pyarrow as pa
 from pyarrow import parquet as pq
 
 
-class TestDaftParquetReader(unittest.TestCase):
+class TestDaftS3FileToTable(unittest.TestCase):
     MVP_PATH = "deltacat/tests/utils/data/mvp.parquet"
 
     def test_read_from_s3_all_columns(self):
@@ -121,5 +121,45 @@ class TestDaftParquetReader(unittest.TestCase):
         self.assertEqual(table.num_rows, 10)
 
 
+class TestDaftS3FilesToDataFrame(unittest.TestCase):
+    MVP_PATH = "deltacat/tests/utils/data/mvp.parquet"
+
+    def test_read_from_s3_all_columns(self):
+        df = s3_files_to_dataframe(
+            uris=[self.MVP_PATH],
+            content_encoding=ContentEncoding.IDENTITY.value,
+            content_type=ContentType.PARQUET.value,
+            ray_init_options={"local_mode": True},
+        )
+
+        table = df.to_arrow()
+        self.assertEqual(table.schema.names, ["a", "b"])
+        self.assertEqual(table.num_rows, 100)
+
+    def test_does_not_read_from_s3_if_not_materialized(self):
+        df = s3_files_to_dataframe(
+            uris=[self.MVP_PATH],
+            content_encoding=ContentEncoding.IDENTITY.value,
+            content_type=ContentType.PARQUET.value,
+            ray_init_options={"local_mode": True},
+        )
+
+        self.assertRaises(RuntimeError, lambda: len(df))
+        df.collect()
+        self.assertEqual(len(df), 100)
+
+    def test_raises_error_if_not_supported_content_type(self):
+
+        self.assertRaises(
+            AssertionError,
+            lambda: s3_files_to_dataframe(
+                uris=[self.MVP_PATH],
+                content_encoding=ContentEncoding.IDENTITY.value,
+                content_type=ContentType.UNESCAPED_TSV.value,
+                ray_init_options={"local_mode": True},
+            ),
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

deltacat/types/media.py

@@ -44,6 +44,11 @@ class TableType(str, Enum):
     PYARROW_PARQUET = "pyarrow_parquet"
 
 
+class DistributedDatasetType(str, Enum):
+    DAFT = "daft"
+    RAY_DATASET = "ray_dataset"
+
+
 class SchemaType(str, Enum):
     ARROW = "arrow"
 

deltacat/types/tables.py

@@ -15,10 +15,11 @@ from ray.data.read_api import (
 )
 
 import deltacat.storage as dcs
-from deltacat.types.media import TableType
+from deltacat.types.media import TableType, DistributedDatasetType
 from deltacat.utils import numpy as np_utils
 from deltacat.utils import pandas as pd_utils
 from deltacat.utils import pyarrow as pa_utils
+from deltacat.utils import daft as daft_utils
 from deltacat.utils.ray_utils import dataset as ds_utils
 
 TABLE_TYPE_TO_READER_FUNC: Dict[int, Callable] = {
@@ -78,6 +79,11 @@ TABLE_TYPE_TO_DATASET_CREATE_FUNC_REFS: Dict[str, Callable] = {
 }
 
 
+DISTRIBUTED_DATASET_TYPE_TO_READER_FUNC: Dict[int, Callable] = {
+    DistributedDatasetType.DAFT.value: daft_utils.s3_files_to_dataframe
+}
+
+
 class TableWriteMode(str, Enum):
     """
     Enum controlling how a given dataset will be written to a table.

deltacat/utils/daft.py

@@ -1,8 +1,9 @@
 import logging
-from typing import Optional, List
-
+from typing import Optional, List, Any, Dict, Callable
+import daft
+import ray
 from daft.table import read_parquet_into_pyarrow
-from daft import TimeUnit
+from daft import TimeUnit, DataFrame
 from daft.io import IOConfig, S3Config
 import pyarrow as pa
 
@@ -22,6 +23,66 @@ from deltacat.types.partial_download import (
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
+def s3_files_to_dataframe(
+    uris: List[str],
+    content_type: str,
+    content_encoding: str,
+    column_names: Optional[List[str]] = None,
+    include_columns: Optional[List[str]] = None,
+    read_func_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    ray_options_provider: Optional[Callable[[int, Any], Dict[str, Any]]] = None,
+    s3_client_kwargs: Optional[Any] = None,
+    ray_init_options: Optional[Dict[str, Any]] = None,
+) -> DataFrame:
+
+    if ray_init_options is None:
+        ray_init_options = {}
+
+    assert (
+        content_type == ContentType.PARQUET.value
+    ), f"daft native reader currently only supports parquet, got {content_type}"
+
+    assert (
+        content_encoding == ContentEncoding.IDENTITY.value
+    ), f"daft native reader currently only supports identity encoding, got {content_encoding}"
+
+    if not ray.is_initialized():
+        ray.init(address="auto", ignore_reinit_error=True, **ray_init_options)
+
+    daft.context.set_runner_ray(noop_if_initialized=True)
+
+    if s3_client_kwargs is None:
+        s3_client_kwargs = {}
+
+    kwargs = {}
+    if read_func_kwargs_provider is not None:
+        kwargs = read_func_kwargs_provider(content_type, kwargs)
+
+    # TODO(raghumdani): pass in coerce_int96_timestamp arg
+    # https://github.com/Eventual-Inc/Daft/issues/1894
+
+    io_config = _get_s3_io_config(s3_client_kwargs=s3_client_kwargs)
+
+    logger.debug(
+        f"Preparing to read S3 object from {len(uris)} files into daft dataframe"
+    )
+
+    df, latency = timed_invocation(
+        daft.read_parquet, path=uris, io_config=io_config, use_native_downloader=True
+    )
+
+    logger.debug(f"Time to create daft dataframe from {len(uris)} files is {latency}s")
+
+    columns_to_read = include_columns or column_names
+
+    logger.debug(f"Taking columns {columns_to_read} from the daft df.")
+
+    if columns_to_read:
+        return df.select(*columns_to_read)
+    else:
+        return df
+
+
 def daft_s3_file_to_table(
     s3_url: str,
     content_type: str,
@@ -55,16 +116,7 @@ def daft_s3_file_to_table(
     ):
         row_groups = partial_file_download_params.row_groups_to_download
 
-    io_config = IOConfig(
-        s3=S3Config(
-            key_id=s3_client_kwargs.get("aws_access_key_id"),
-            access_key=s3_client_kwargs.get("aws_secret_access_key"),
-            session_token=s3_client_kwargs.get("aws_session_token"),
-            retry_mode="adaptive",
-            num_tries=BOTO_MAX_RETRIES,
-            max_connections=DAFT_MAX_S3_CONNECTIONS_PER_FILE,
-        )
-    )
+    io_config = _get_s3_io_config(s3_client_kwargs=s3_client_kwargs)
 
     logger.debug(f"Preparing to read S3 object from {s3_url} into daft table")
 
@@ -95,3 +147,16 @@ def daft_s3_file_to_table(
         return coerce_pyarrow_table_to_schema(pa_table, input_schema)
     else:
         return pa_table
+
+
+def _get_s3_io_config(s3_client_kwargs) -> IOConfig:
+    return IOConfig(
+        s3=S3Config(
+            key_id=s3_client_kwargs.get("aws_access_key_id"),
+            access_key=s3_client_kwargs.get("aws_secret_access_key"),
+            session_token=s3_client_kwargs.get("aws_session_token"),
+            retry_mode="adaptive",
+            num_tries=BOTO_MAX_RETRIES,
+            max_connections=DAFT_MAX_S3_CONNECTIONS_PER_FILE,
+        )
+    )

{deltacat-0.2.10.dist-info → deltacat-1.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 0.2.10
+Version: 1.0.0
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -27,7 +27,7 @@ Requires-Dist: tenacity ==8.1.0
 Requires-Dist: typing-extensions ==4.4.0
 Requires-Dist: pymemcache ==4.0.0
 Requires-Dist: redis ==4.6.0
-Requires-Dist: getdaft ==0.2.13
+Requires-Dist: getdaft ==0.2.16
 Requires-Dist: schedule ==1.2.0
 
 # DeltaCAT

{deltacat-0.2.10.dist-info → deltacat-1.0.0.dist-info}/RECORD

@@ -1,11 +1,11 @@
-deltacat/__init__.py,sha256=HQxIkU4btF_x_4uymlZWtg9axgY7s2D7JzQzFScu1RU,1778
+deltacat/__init__.py,sha256=hul5a8v-ltL75fkLOfQBj_i2OoY5tKMy-yO3oTZzikA,1777
 deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
 deltacat/exceptions.py,sha256=xqZf8CwysNYP2d39pf27OnXGStPREgBgIM-e2Tts-TI,199
 deltacat/logs.py,sha256=9XWuTBoWhhAF9rAL6t9veXmnAlJHsaqk0lTxteVPqyQ,5674
 deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/aws/clients.py,sha256=wWiqXyZPWXezdEbhQ7DLwEVnYV6KiitqzBc5B4UAwc0,6184
 deltacat/aws/constants.py,sha256=luXWMO_8eatq8f9NlFjNM7q362j77JwzTM2BEVS_8-8,353
-deltacat/aws/s3u.py,sha256=s2On5X3IQiCsCMKw4lpfV1GfKQVWOXNsdAmIJK5PEM0,18610
+deltacat/aws/s3u.py,sha256=aK1_pyfipd9Jq1ZiaOC-gszyIIfc1TSbch6YmuZmjt0,23878
 deltacat/aws/redshift/__init__.py,sha256=7SvjG-dqox8zZUhFicTsUvpG5vXYDl_QQ3ohlHOgTKc,342
 deltacat/aws/redshift/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/aws/redshift/model/manifest.py,sha256=ThgpdwzaWz493Zz9e8HSWwuxEheA1nDuypM3pe4vozk,12987
@@ -15,6 +15,7 @@ deltacat/benchmarking/conftest.py,sha256=6M9NJ71vnOpeMxG-Ly9UWRsgZmky5-1GTuoRD-O
 deltacat/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/catalog/delegate.py,sha256=lVPPG4bBxoNt0IFufKIXc7H2Nhvb6QNox-WfXxaNxuc,8745
 deltacat/catalog/interface.py,sha256=On5hsbznnIKsvCcm5C1N3OS3ar3v6q7pAUkLWMb6PtY,6556
+deltacat/catalog/default_catalog_impl/__init__.py,sha256=ted1_sA2Y3ljJjb9hRuWUCiUlCen-HAW8TxoV4g4IW0,12677
 deltacat/catalog/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/catalog/model/catalog.py,sha256=-Ho7a3rV1hiOS9cSRCAor9AtXV9nJn9t_MDVql9pIxo,2212
 deltacat/catalog/model/table_definition.py,sha256=tKrM1mmaQlvxqXrLt3QJVZK5BZfaJnhjTZ6KjybYlhE,727
@@ -48,22 +49,31 @@ deltacat/compute/compactor/utils/round_completion_file.py,sha256=DmZfHeAXlQn0DDd
 deltacat/compute/compactor/utils/sort_key.py,sha256=oK6otg-CSsma6zlGPaKg-KNEvcZRG2NqBlCw1X3_FBc,2397
 deltacat/compute/compactor/utils/system_columns.py,sha256=CNIgAGos0xAGEpdaQIH7KfbSRrGZgjRbItXMararqXQ,9399
 deltacat/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/compaction_session.py,sha256=YnKG2LlrgYYsVKt_6txcXzCgolaQWF4SuQz0eZmChZM,20422
+deltacat/compute/compactor_v2/compaction_session.py,sha256=QVWuI5edQORqhBkccM5waEXMOzg_6756-eT4YqFxkD4,21932
 deltacat/compute/compactor_v2/constants.py,sha256=yZgzFD59wiXbXiTVgYPWRodZGpngiSBNFB2jmoZ4fps,1471
 deltacat/compute/compactor_v2/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/model/hash_bucket_input.py,sha256=pgE2o8Z9-Dvs75C15LAkmfuJFFi5pRIuuxA9GGyDlLM,2631
+deltacat/compute/compactor_v2/model/hash_bucket_input.py,sha256=HS1BLgNCjQoqe7EuM9baQWE6U4BFqU2PVEWFEF725VE,2815
 deltacat/compute/compactor_v2/model/hash_bucket_result.py,sha256=EsY9BPPywhmxlcLKn3kGWzAX4s4BTR2vYyPUB-wAEOc,309
-deltacat/compute/compactor_v2/model/merge_input.py,sha256=A-_Oq54sx1vrT-Ewv2_yKARdIh928yJvEuheCkw5tvQ,5049
-deltacat/compute/compactor_v2/model/merge_result.py,sha256=L53i9iL_XpzqBr7HETixD5v5qfLvitkGcjoML_hHfcs,368
+deltacat/compute/compactor_v2/model/merge_file_group.py,sha256=1o86t9lc3K6ZvtViVO1SVljCj6f0B3MfB3hqtGm2S0s,7410
+deltacat/compute/compactor_v2/model/merge_input.py,sha256=xiUsA2jY1tQRGc84Q2SygijF5N47x3WqFjcy6sJwEIs,4511
+deltacat/compute/compactor_v2/model/merge_result.py,sha256=R9PV-0NGJ9GaSSeVqbIEhiqBolshPP4m7_eCb0yJR8g,401
 deltacat/compute/compactor_v2/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=wFu4vAS8PR0_SxxLIfGPmtLjUV9hCfPeHG56CFpoLIM,8100
-deltacat/compute/compactor_v2/steps/merge.py,sha256=QI8ovaO6yPw_VgDYqTzQOxw2oov4ipuW2gR-w01FWGY,18087
+deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=2RfkVh-0mccgONa4atQFmNfPk668oIZr8TthwTUyh-k,6295
+deltacat/compute/compactor_v2/steps/merge.py,sha256=UR6HbPUNxEVtvsGiibRYmzq0_nT9Lq6AbEa6e71KaNQ,14496
 deltacat/compute/compactor_v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/utils/content_type_params.py,sha256=rNKZisxGrLQOkwX8eHUQiFoTR1V-E66pMqWigtrs618,2156
 deltacat/compute/compactor_v2/utils/dedupe.py,sha256=62tFCY2iRP7I3-45GCIYs6_SJsQl8C5lBEr8gbNfbsw,1932
+deltacat/compute/compactor_v2/utils/delta.py,sha256=73wET1zqQyYR8eaO59XyKYVn-lqshPrScSiiFnHRUj4,3373
 deltacat/compute/compactor_v2/utils/io.py,sha256=jgIfwrfH2mTFUx1M0TgwZGGfrS4IXjP1PmqwaQmNAJM,5092
+deltacat/compute/compactor_v2/utils/merge.py,sha256=tcHlaHHeEN1F_rrAadS_TPWX719uGKjE8yTeVwEwsTk,4843
 deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=MAscmL35WfwN7Is72aFlD_cGhxtZgjRwwR5kS9Yn2uU,11393
-deltacat/compute/compactor_v2/utils/task_options.py,sha256=LA1QbiDv3f9LJQwjKz3-YH3TpK3exL1c5acaGAOF57E,10210
+deltacat/compute/compactor_v2/utils/task_options.py,sha256=Ndhff9F_zff6zX3mw4AztHkvSBgR4O8SgMUgwq3rvyM,10601
+deltacat/compute/merge_on_read/__init__.py,sha256=ckbgngmqPjYBYz_NySsR1vNTOb_hNpeL1sYkZKvBI9M,214
+deltacat/compute/merge_on_read/daft.py,sha256=1oC38u5ig_aTrq7EzyWBo8Ui54rb6yERYMk-vEFbpxM,1400
+deltacat/compute/merge_on_read/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/merge_on_read/model/merge_on_read_params.py,sha256=Q51znagh8PtLnsY987Ulx9n20oAydfPq3Zd3Y9ocbTI,2035
+deltacat/compute/merge_on_read/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/merge_on_read/utils/delta.py,sha256=e4BtOHa5XPpUnR4r0HqBKjXckBsTI8qBwdUWwpJfkWQ,1367
 deltacat/compute/metastats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/metastats/meta_stats.py,sha256=78hN3aN5wLHUFJsZXuv2JLeqA35HZ8mLUWJDMslMj5Q,18731
 deltacat/compute/metastats/stats.py,sha256=8iUiSXOAjqiEeNP5RIb5gvhykBgpNHD5IKkB8zsPR0E,7363
@@ -102,7 +112,7 @@ deltacat/io/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/aws/redshift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/aws/redshift/redshift_datasource.py,sha256=X183O4tgBqtaZOSFmMFvp-9mv8NX5kGvRvX0eoSX8rA,22599
 deltacat/storage/__init__.py,sha256=B2cjcqySR6g0LBK83H0nOVojYNurbhp5kq3n8pfkzcA,1466
-deltacat/storage/interface.py,sha256=JEtJ1zzsPUEEo3_K5vUeCN4nQDbPyJ6neHllfYt4tCk,21347
+deltacat/storage/interface.py,sha256=BsB8YnsB-RdUMJSmHj6ef-qBOj5yqWoU9b0HKzQT8OM,21569
 deltacat/storage/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/storage/model/delta.py,sha256=2qhCiEWmrybV0mfBTKHi9VKlNTqJ6wkE5ssq4ozMMQQ,13757
 deltacat/storage/model/list_result.py,sha256=FgD6oYeKo0EPe8z7jC8T4pAFjBOuBwd4axxGrnYyBG4,2466
@@ -113,15 +123,17 @@ deltacat/storage/model/sort_key.py,sha256=SPIxJfI_o7fbp1s3ZKMyX9x7_jK8UZapaVnKSA
 deltacat/storage/model/stream.py,sha256=XZ-c4EQR89NWydEOEG5GCaT8zST10OmjLZBKHZPdrzA,7738
 deltacat/storage/model/table.py,sha256=IOu1ZOrdRkVDB-FOxYMRvnNf5TukIDfbdHWTqHYN_OY,4225
 deltacat/storage/model/table_version.py,sha256=cOM9dN-YB_Hhi4h1CzFbldC5qRkm4C1rQ3rpKIZzCNs,7413
-deltacat/storage/model/types.py,sha256=8SuReHxWtGLomac8eTibzJcNYfLansZHu-CxWI6Wwno,2114
+deltacat/storage/model/types.py,sha256=hj7MmjjVmKT-R9sMUulOWG-FByGZKKaYXNnOWW32mP0,1608
 deltacat/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/aws/test_clients.py,sha256=23GMWfz27WWBDXSqphG9mfputsyS7j3I5P_HRk4YoKE,3790
+deltacat/tests/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/tests/catalog/test_default_catalog_impl.py,sha256=9srCU5yQ159oZ9_PoJ_mWMzVUW5bKV0mnmPJc5zKCQQ,3125
 deltacat/tests/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/compute/compact_partition_test_cases.py,sha256=EyZwh-7qKiMmzJT8E_V74cvle5uONYZyt89jmdAu1TI,47952
-deltacat/tests/compute/test_compact_partition_incremental.py,sha256=-nIQev0FYWbp76LwM0H4KpMEgP2GMqVRFFJHMsLUN2E,10011
+deltacat/tests/compute/compact_partition_test_cases.py,sha256=nvFIEQWvRjpi5yaYL6KIGKgBUjKqYE_elCcc3GHHUlU,53188
+deltacat/tests/compute/test_compact_partition_incremental.py,sha256=F02ia0zvapQmSJ48KGdwhtjaUrIFQmH77CKNLINUsWQ,10102
 deltacat/tests/compute/test_compact_partition_params.py,sha256=MIzIcBscwFA1W-cfTTxVx0zcgbrs8D4bI9Hy4TF5eRo,8322
-deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=-yFmEGqWMTIq9iShFU9rn4cX7ky1Zmm3pv4F9NwsQUo,13218
+deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=s17wb7ub0v_1qLhIzLaHZ68P2xdGilQlV8ibqj57PCc,13372
 deltacat/tests/compute/test_util_common.py,sha256=Skz0ZfHzidArZhIzRDHOYt-5uGBwx6MRfKZpeBnzh9w,6055
 deltacat/tests/compute/test_util_constant.py,sha256=4o-W3E7r7jhFl1A3OFLLrdKnwcF46zx4lEIDY8ONJ3c,929
 deltacat/tests/compute/test_util_create_table_deltas_repo.py,sha256=5yP285lY539CP1UuyYe8Kz14CnBUpE1kZJZjxBAaXew,6530
@@ -142,30 +154,30 @@ deltacat/tests/io/test_memcached_object_store.py,sha256=gUVYycPkNpq9XxotdJwFZ2HO
 deltacat/tests/io/test_ray_plasma_object_store.py,sha256=-wJZP6lRtEOogR25wjEiIBGz_lpvWVihwlZ5GqandZU,1911
 deltacat/tests/io/test_redis_object_store.py,sha256=sZrXrYjkw8u_XrvFilhBbLc8PPnZiuMKa1_Bt9ka5qs,3838
 deltacat/tests/io/test_s3_object_store.py,sha256=4b7PYEfQJnYGUz6fcLFWVVyRHTlH_yd8CIaCv9l33Gg,1900
-deltacat/tests/local_deltacat_storage/__init__.py,sha256=Jv0ZVjEvFptoNTZoIFvFYisjkqY4O5HSnviUPbiTQUY,34776
+deltacat/tests/local_deltacat_storage/__init__.py,sha256=UGf8W9y2gDgOTR2Uj9qWCqe7CXu0GTl1hygoSFEKe80,35341
 deltacat/tests/stats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/stats/test_intervals.py,sha256=S92DgkALQ1WmbLWcxtvS7RlVGvL-XoPJKUUbkdn9_CQ,1955
 deltacat/tests/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/test_utils/constants.py,sha256=zgqFmfIE5ZCtDw4NF-Y4ZEEnaPUP5nDY5768WPod0Fc,208
-deltacat/tests/test_utils/pyarrow.py,sha256=QUqiEbHC3_v0BqDgQyHWxyCrsmqXlz6CI5lJPs-3x7I,1510
+deltacat/tests/test_utils/pyarrow.py,sha256=nIvS01vJbUMeJ_CoNEmylL5Ca-r9X8M3NlbV0OBnDOs,2000
 deltacat/tests/test_utils/storage.py,sha256=93GEn4A5WbMHWk0Ec4Bd7RxeHoSEnBfSarfWhKOSNtM,972
 deltacat/tests/test_utils/utils.py,sha256=a32qEwcSSd1lvRi0aJJ4ZLnc1ZyXmoQF_K95zaQRk2M,455
 deltacat/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/utils/test_cloudpickle.py,sha256=J0pnBY3-PxlUh6MamZAN1PuquKQPr2iyzjiJ7-Rcl0o,1506
-deltacat/tests/utils/test_daft.py,sha256=iN6rAwGXw5F4xT2UZ72bN276hkKVD7XD4WNp5DKgm2Q,5098
+deltacat/tests/utils/test_daft.py,sha256=Xal84zR42rXsWQI3lImdDYWOzewomKmhmiUQ59m67V0,6488
 deltacat/tests/utils/test_pyarrow.py,sha256=eZAuYp9MUf8lmpIilH57JkURuNsTGZ3IAGC4Gm5hdrM,17307
 deltacat/tests/utils/test_record_batch_tables.py,sha256=AkG1WyljQmjnl-AxhbFWyo5LnMIKRyLScfgC2B_ES-s,11321
 deltacat/tests/utils/test_resources.py,sha256=HtpvDrfPZQNtGDXUlsIzc_yd7Vf1cDscZ3YbN0oTvO8,2560
 deltacat/tests/utils/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/types/media.py,sha256=RALwafQ0SwMyPUIcENhURk7Sor_2CIfEMztvFUnvZFQ,2227
+deltacat/types/media.py,sha256=7_QRU6NbjmJk0GLAn_Km6ja8RE5G3V8jvLfUXqnjnqU,2320
 deltacat/types/partial_download.py,sha256=9BJ5b0DHyWWeV7wMZjOfYoeH_iil_bjZ9b_WMpUzvHs,2516
-deltacat/types/tables.py,sha256=xedkualOnncyH_AjiflrkQY4YCZ_IW-zdOZRsHm8_5I,4198
+deltacat/types/tables.py,sha256=HLm-xI5xCuKTeGbaErspni3DeSVC_aRe2TYFLd6b_mA,4409
 deltacat/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/utils/arguments.py,sha256=5y1Xz4HSAD8M8Jt83i6gOEKoYjy_fMQe1V43IhIE4hY,1191
 deltacat/utils/cloudpickle.py,sha256=XE7YDmQe56ksfl3NdYZkzOAhbHSuhNcBZGOehQpgZr0,1187
 deltacat/utils/common.py,sha256=RG_-enXNpLKaYrqyx1ne2lL10lxN9vK7F631oJP6SE8,1375
-deltacat/utils/daft.py,sha256=eZG1AjK21lM7bzEc3_BniDqpqMGDrlp_qj9Du4dxaV0,3334
+deltacat/utils/daft.py,sha256=UlbJpWsxZQ1pC0RVks5lsVJk3Bq5pa0Rw2lKMc9Gz4Q,5441
 deltacat/utils/metrics.py,sha256=Ob-RXGoNnfTMRXaNbSHoqW8y-n8KfRA9nLuo9AvsReI,6201
 deltacat/utils/numpy.py,sha256=ZiGREobTVT6IZXgPxkSUpLJFN2Hn8KEZcrqybLDXCIA,2027
 deltacat/utils/pandas.py,sha256=eGOpiZE1zLznTtuwoN80j4PBp1_bUV8SE4c951r0a3o,9561
@@ -181,8 +193,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=SIljK3UkSqQ6Ntit_iSiYt9yYjN_gGrCTX6_72XdQ3w,3244
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=xOVkqL6o8qGsewGvzhMKxmCcqcFZDnNILuz5IGMgxSc,4991
-deltacat-0.2.10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-0.2.10.dist-info/METADATA,sha256=ZI6_ysr3AKmzAUGDPkypTm1QhtvGzVK3Yxnuu6KgM4Q,1781
-deltacat-0.2.10.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-deltacat-0.2.10.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-0.2.10.dist-info/RECORD,,
+deltacat-1.0.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.0.0.dist-info/METADATA,sha256=Jtp8Vk3_u99IPtRxgJ59xSXCWk3GPF2Uk4tX9x-tUcM,1780
+deltacat-1.0.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+deltacat-1.0.0.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.0.0.dist-info/RECORD,,