deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/merge_on_read/model/merge_on_read_params.py

@@ -1,66 +0,0 @@
-from __future__ import annotations
-from typing import Optional, Dict, List, Union, Any
-from deltacat.storage import (
-    Delta,
-    DeltaLocator,
-    interface as unimplemented_deltacat_storage,
-)
-
-
-class MergeOnReadParams(dict):
-    """
-    This class represents the parameters passed to compact_partition (deltacat/compute/compactor/compaction_session.py)
-    """
-
-    @staticmethod
-    def of(params: Optional[Dict]) -> MergeOnReadParams:
-        params = {} if params is None else params
-
-        result = MergeOnReadParams(params)
-        assert result.deltas is not None, "deltas is a required arg"
-
-        result.deltacat_storage = params.get(
-            "deltacat_storage", unimplemented_deltacat_storage
-        )
-        result.reader_kwargs = params.get("reader_kwargs", {})
-        result.deltacat_storage_kwargs = params.get("deltacat_storage_kwargs", {})
-
-        return result
-
-    @property
-    def deltas(self) -> List[Union[Delta, DeltaLocator]]:
-        """
-        The list of deltas to compact in-memory.
-        """
-        return self["deltas"]
-
-    @deltas.setter
-    def deltas(self, to_set: List[Union[Delta, DeltaLocator]]) -> None:
-        self["deltas"] = to_set
-
-    @property
-    def reader_kwargs(self) -> Dict[Any, Any]:
-        """
-        The key word arguments to be passed to the reader.
-        """
-        return self["reader_kwargs"]
-
-    @reader_kwargs.setter
-    def reader_kwargs(self, kwargs: Dict[Any, Any]) -> None:
-        self["reader_kwargs"] = kwargs
-
-    @property
-    def deltacat_storage(self) -> unimplemented_deltacat_storage:
-        return self["deltacat_storage"]
-
-    @deltacat_storage.setter
-    def deltacat_storage(self, storage: unimplemented_deltacat_storage) -> None:
-        self["deltacat_storage"] = storage
-
-    @property
-    def deltacat_storage_kwargs(self) -> dict:
-        return self["deltacat_storage_kwargs"]
-
-    @deltacat_storage_kwargs.setter
-    def deltacat_storage_kwargs(self, kwargs: dict) -> None:
-        self["deltacat_storage_kwargs"] = kwargs

deltacat/compute/merge_on_read/utils/delta.py

@@ -1,42 +0,0 @@
-from typing import List, Dict, Any, Optional, Union
-from deltacat.storage.model.delta import Delta, DeltaLocator
-from deltacat.storage.model.types import DistributedDataset
-from deltacat.storage import (
-    interface as unimplemented_deltacat_storage,
-)
-from deltacat.types.media import TableType, StorageType, DistributedDatasetType
-
-
-def create_df_from_all_deltas(
-    deltas: List[Union[Delta, DeltaLocator]],
-    table_type: TableType,
-    distributed_dataset_type: DistributedDatasetType,
-    reader_kwargs: Optional[Dict[Any, Any]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[Dict[Any, Any]] = None,
-    *args,
-    **kwargs
-) -> List[DistributedDataset]:  # type: ignore
-    """
-    This method creates a distributed dataset for each delta and returns their references.
-    """
-
-    if reader_kwargs is None:
-        reader_kwargs = {}
-    if deltacat_storage_kwargs is None:
-        deltacat_storage_kwargs = {}
-
-    df_list = []
-
-    for delta in deltas:
-        df = deltacat_storage.download_delta(
-            delta_like=delta,
-            table_type=table_type,
-            distributed_dataset_type=distributed_dataset_type,
-            storage_type=StorageType.DISTRIBUTED,
-            **reader_kwargs,
-            **deltacat_storage_kwargs
-        )
-        df_list.append(df)
-
-    return df_list

deltacat/examples/common/fixtures.py

@@ -1,15 +0,0 @@
-import os
-import logging
-import argparse
-from deltacat import logs
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-
-def store_cli_args_in_os_environ(script_args_list=[]):
-    parser = argparse.ArgumentParser()
-    for args, kwargs in script_args_list:
-        parser.add_argument(*args, **kwargs)
-    args = parser.parse_args()
-    print(f"Command Line Arguments: {args}")
-    os.environ.update(vars(args))

deltacat/storage/iceberg/iceberg_scan_planner.py

@@ -1,28 +0,0 @@
-from typing import Optional
-
-from pyiceberg.catalog import Catalog
-from deltacat.storage.model.scan.push_down import Pushdown
-from deltacat.storage.model.scan.scan_plan import ScanPlan
-from deltacat.storage.model.scan.scan_task import FileScanTask, DataFile
-from deltacat.storage.util.scan_planner import ScanPlanner
-from deltacat.storage.iceberg.impl import _try_load_iceberg_table
-
-
-class IcebergScanPlanner(ScanPlanner):
-    def __init__(self, catalog: Catalog):
-        self.catalog = catalog
-
-    def create_scan_plan(
-        self,
-        table_name: str,
-        namespace: Optional[str] = None,
-        pushdown: Optional[Pushdown] = None,
-    ) -> ScanPlan:
-        iceberg_table = _try_load_iceberg_table(
-            self.catalog, namespace=namespace, table_name=table_name
-        )
-        file_scan_tasks = []
-        # TODO: implement predicate pushdown to Iceberg
-        for scan_task in iceberg_table.scan().plan_files():
-            file_scan_tasks.append(FileScanTask([DataFile(scan_task.file.file_path)]))
-        return ScanPlan(file_scan_tasks)

deltacat/storage/rivulet/__init__.py

@@ -1,11 +0,0 @@
-from .schema.schema import Schema
-from .schema.schema import Field
-from .dataset import Dataset
-from .schema.schema import Datatype
-
-__all__ = [
-    "Schema",
-    "Field",
-    "Dataset",
-    "Datatype",
-]

deltacat/storage/rivulet/feather/__init__.py

@@ -1,5 +0,0 @@
-# TODO later on this will be moved to a dedicated package
-from deltacat.storage.rivulet.feather.file_reader import FeatherFileReader
-from deltacat.storage.rivulet.reader.reader_type_registrar import FileReaderRegistrar
-
-FileReaderRegistrar.register_reader("feather", FeatherFileReader)

deltacat/storage/rivulet/parquet/__init__.py

@@ -1,5 +0,0 @@
-# TODO later on this will be moved to a dedicated package
-from deltacat.storage.rivulet.parquet.file_reader import ParquetFileReader
-from deltacat.storage.rivulet.reader.reader_type_registrar import FileReaderRegistrar
-
-FileReaderRegistrar.register_reader("parquet", ParquetFileReader)

deltacat/tests/compute/compactor/utils/test_round_completion_file.py

@@ -1,231 +0,0 @@
-import pytest
-import os
-from moto import mock_s3
-import boto3
-from boto3.resources.base import ServiceResource
-from deltacat.compute.compactor.utils.round_completion_file import (
-    read_round_completion_file,
-    write_round_completion_file,
-)
-from deltacat.tests.compute.test_util_common import get_test_partition_locator
-from deltacat.compute.compactor import RoundCompletionInfo
-
-RCF_BUCKET_NAME = "rcf-bucket"
-
-
-@pytest.fixture(autouse=True, scope="module")
-def mock_aws_credential():
-    os.environ["AWS_ACCESS_KEY_ID"] = "testing"
-    os.environ["AWS_SECRET_ACCESS_ID"] = "testing"
-    os.environ["AWS_SECURITY_TOKEN"] = "testing"
-    os.environ["AWS_SESSION_TOKEN"] = "testing"
-    os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
-    yield
-
-
-@pytest.fixture(autouse=True, scope="module")
-def s3_resource(mock_aws_credential):
-    with mock_s3():
-        yield boto3.resource("s3")
-
-
-@pytest.fixture(autouse=True, scope="function")
-def setup_compaction_artifacts_s3_bucket(s3_resource: ServiceResource):
-    s3_resource.create_bucket(
-        ACL="authenticated-read",
-        Bucket=RCF_BUCKET_NAME,
-    )
-    yield
-    s3_resource.Bucket(RCF_BUCKET_NAME).objects.all().delete()
-
-
-class TestReadWriteRoundCompletionFile:
-    def test_read_when_rcf_written_without_destination(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-        destination_locator = get_test_partition_locator("destination")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        rcf_url = write_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, None, expected_rcf
-        )
-
-        rcf = read_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator
-        )
-
-        assert (
-            rcf_url == "s3://rcf-bucket/f9829af39770d904dbb811bd8f4e886dd307f507.json"
-        )
-        assert rcf == expected_rcf
-
-    def test_read_when_rcf_written_with_destination(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-        destination_locator = get_test_partition_locator("destination")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        rcf_url = write_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator, expected_rcf
-        )
-
-        rcf = read_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator
-        )
-
-        assert (
-            rcf_url
-            == "s3://rcf-bucket/f9829af39770d904dbb811bd8f4e886dd307f507/e9939deadc091b3289a2eb0ca56b1ba86b9892f4.json"
-        )
-        assert rcf == expected_rcf
-
-    def test_read_without_destination_when_rcf_written_with_destination(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-        destination_locator = get_test_partition_locator("destination")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        write_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator, expected_rcf
-        )
-
-        rcf = read_round_completion_file(RCF_BUCKET_NAME, source_locator, None)
-
-        assert rcf is None
-
-    def test_read_without_destination_when_rcf_written_without_destination(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        write_round_completion_file(RCF_BUCKET_NAME, source_locator, None, expected_rcf)
-
-        rcf = read_round_completion_file(RCF_BUCKET_NAME, source_locator, None)
-
-        assert rcf == expected_rcf
-
-    def test_read_when_rcf_written_both_with_and_without_destination(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-        destination_locator = get_test_partition_locator("destination")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        expected_rcf_2 = RoundCompletionInfo.of(
-            high_watermark=1223,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=1233,
-        )
-
-        write_round_completion_file(RCF_BUCKET_NAME, source_locator, None, expected_rcf)
-
-        write_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator, expected_rcf_2
-        )
-
-        rcf = read_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator
-        )
-
-        assert rcf == expected_rcf_2
-
-    def test_read_when_none_destination_partition_id(self):
-
-        source_locator = get_test_partition_locator("source")
-        destination_locator = get_test_partition_locator(None)
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        write_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator, expected_rcf
-        )
-
-        rcf = read_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator
-        )
-
-        assert rcf == expected_rcf
-
-    def test_write_when_custom_url_is_passed(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        completion_file_s3_url = f"s3://{RCF_BUCKET_NAME}/test.json"
-        rcf_url = write_round_completion_file(
-            RCF_BUCKET_NAME,
-            source_locator,
-            None,
-            expected_rcf,
-            completion_file_s3_url=completion_file_s3_url,
-        )
-
-        rcf = read_round_completion_file(RCF_BUCKET_NAME, source_locator, None)
-
-        assert rcf_url == completion_file_s3_url
-        assert rcf is None