deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/io/read_api.py

@@ -1,143 +0,0 @@
-from typing import Any, Callable, Dict, List, Optional, Union
-
-import pyarrow as pa
-import s3fs
-from ray.data import read_datasource
-from ray.data._internal.arrow_block import ArrowRow
-
-from deltacat import ContentType
-from deltacat.io.aws.redshift.redshift_datasource import (
-    HivePartitionParser,
-    RedshiftDatasource,
-    RedshiftUnloadTextArgs,
-    S3PathType,
-)
-from deltacat.io.dataset import DeltacatDataset
-from deltacat.utils.common import ReadKwargsProvider
-
-
-def read_redshift(
-    paths: Union[str, List[str]],
-    *,
-    path_type: S3PathType = S3PathType.MANIFEST,
-    filesystem: Optional[Union[pa.fs.S3FileSystem, s3fs.S3FileSystem]] = None,
-    columns: Optional[List[str]] = None,
-    schema: Optional[pa.Schema] = None,
-    unload_text_args: RedshiftUnloadTextArgs = RedshiftUnloadTextArgs(),
-    partitioning: HivePartitionParser = None,
-    content_type_provider: Callable[[str], ContentType] = lambda p: ContentType.PARQUET
-    if p.endswith(".parquet")
-    else ContentType.CSV,
-    parallelism: int = 200,
-    ray_remote_args: Dict[str, Any] = None,
-    arrow_open_stream_args: Optional[Dict[str, Any]] = None,
-    pa_read_func_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    **kwargs,
-) -> DeltacatDataset[ArrowRow]:
-    """Reads Redshift UNLOAD results from either S3 Parquet or delimited text
-    files into a Ray Dataset.
-
-    Examples:
-        >>> # Read all files contained in a Redshift Manifest:
-        >>> import deltacat as dc
-        >>> dc.io.read_redshift("/bucket/dir/manifest")
-
-        >>> # Read all files matching the given key prefix. If this prefix
-        >>> # refers to multiple files, like s3://bucket/data.parquet,
-        >>> # s3://bucket/data.1.csv, etc. then all will be read. The dataset
-        >>> # schema will be inferred from the first parquet file and used for
-        >>> # explicit type conversion of all CSV files:
-        >>> dc.io.read_redshift(
-        >>>     "s3://bucket/data.txt",
-        >>>     path_type=S3PathType.PREFIX)
-
-        >>> # Read all files matching the given key prefix. If this prefix
-        >>> # refers to multiple files or folders, like s3://bucket/dir/,
-        >>> # s3://bucket/dir1/, s3://bucket/dir.txt, s3://bucket/dir.txt.1,
-        >>> # then all files and subfolder contents will be read.
-        >>> dc.io.read_redshift(
-        >>>     "/bucket/dir",
-        >>>     path_type=S3PathType.PREFIX)
-
-        >>> # Read multiple files and folders:
-        >>> dc.io.read_redshift(
-        >>>     ["/bucket/file1", "/bucket/folder1/"],
-        >>>     path_type=S3PathType.FILES_AND_FOLDERS)
-
-        >>> # Read multiple Parquet and CSV files. The dataset schema will be
-        >>> # inferred from the first parquet file and used for explicit type
-        >>> # conversion of all CSV files:
-        >>> dc.io.read_redshift(
-        >>>     ["/bucket/file.parquet", "/bucket/file.csv"],
-        >>>     path_type=S3PathType.FILES_AND_FOLDERS)
-
-    Args:
-        paths: Paths to S3 files and folders to read. If `path_type` is
-            `MANIFEST` then this must be an S3 Redshift Manifest JSON file. If
-            `path_type` is `PREFIX` then this must be a valid S3 key prefix.
-            All files matching the key prefix, including files in matching
-            subdirectories, will be read. Unless custom
-            `content_type_extensions` are specified, file content types will be
-            inferred by file extension with ".parquet" used for Parquet files,
-            and all others assumed to be delimited text (e.g. CSV). It's
-            recommended to specify the path to a manifest unloaded with the
-            VERBOSE option whenever possible to improve the correctness and
-            performance of Dataset reads, compute operations, and writes.
-            `FILES_AND_FOLDERS` is not recommended when reading thousands of
-            files due to its relatively high-latency.
-        path_type: Determines how the `paths` parameter is interpreted.
-        filesystem: The filesystem implementation to read from. This should be
-            either PyArrow's S3FileSystem or s3fs.
-        columns: A list of column names to read. Reads all columns if None or
-            empty.
-        schema: PyArrow schema used to determine delimited text column
-            names and types. If not specified and both Parquet and delimited
-            text files are read as input, then the first Parquet file schema
-            discovered is used instead.
-        unload_text_args: Arguments used when running Redshift `UNLOAD` to
-            text file formats (e.g. CSV). These arguments ensure that all input
-            text files will be correctly parsed. If not specified, then all
-            text files read are assumed to use Redshift UNLOAD's default
-            pipe-delimited text format.
-        partition_base_dir: Base directory to start searching for partitions
-            (exclusive). File paths outside of this directory will not be parsed
-            for partitions and automatically added to the dataset without passing
-            through any partition filter. Specify `None` or an empty string to
-            search for partitions in all file path directories.
-        partition_filter_fn: Callback used to filter `PARTITION` columns. Receives a
-            dictionary mapping partition keys to values as input, returns `True` to
-            read a partition, and `False` to skip it. Each partition key and value
-            is a string parsed directly from an S3 key using hive-style
-            partition directory names of the form "{key}={value}". For example:
-            ``lambda x:
-            True if x["month"] == "January" and x["year"] == "2022" else False``
-        content_type_provider: Takes a file path as input and returns the file
-            content type as output.
-        parallelism: The requested parallelism of the read. Parallelism may be
-            limited by the number of files of the dataset.
-        ray_remote_args: kwargs passed to `ray.remote` in the read tasks.
-        arrow_open_stream_args: kwargs passed to to
-            `pa.fs.open_input_stream()`.
-        pa_read_func_kwargs_provider: Callback that takes a `ContentType` value
-            string as input, and provides read options to pass to either
-            `pa.csv.open_csv()` or `pa.parquet.read_table()` as output.
-    Returns:
-        Dataset holding Arrow records read from the specified paths.
-    """
-    dataset = read_datasource(
-        RedshiftDatasource(),
-        parallelism=parallelism,
-        paths=paths,
-        content_type_provider=content_type_provider,
-        path_type=path_type,
-        filesystem=filesystem,
-        columns=columns,
-        schema=schema,
-        unload_args=unload_text_args,
-        partitioning=partitioning,
-        ray_remote_args=ray_remote_args,
-        open_stream_args=arrow_open_stream_args,
-        read_kwargs_provider=pa_read_func_kwargs_provider,
-        **kwargs,
-    )
-    return DeltacatDataset.from_dataset(dataset)

deltacat/storage/model/delete_parameters.py

@@ -1,40 +0,0 @@
-# Allow classes to use self-referencing Type hints in Python 3.7.
-from __future__ import annotations
-
-from typing import List, Optional
-
-
-class DeleteParameters(dict):
-    """
-    Contains all parameters required to support DELETEs
-    equality_column_names: List of column names that would be used to determine row equality for equality deletes.  Relevant only to equality deletes
-    """
-
-    @staticmethod
-    def of(
-        equality_column_names: Optional[List[str]] = None,
-    ) -> DeleteParameters:
-        delete_parameters = DeleteParameters()
-        if equality_column_names is not None:
-            delete_parameters["equality_column_names"] = equality_column_names
-        return delete_parameters
-
-    @property
-    def equality_column_names(self) -> Optional[List[str]]:
-        return self.get("equality_column_names")
-
-    @staticmethod
-    def merge_delete_parameters(
-        delete_parameters: List[DeleteParameters],
-    ) -> Optional[DeleteParameters]:
-        if len(delete_parameters) < 2:
-            return delete_parameters
-        equality_column_names = delete_parameters[0].equality_column_names
-        assert all(
-            delete_prev.equality_column_names == delete_curr.equality_column_names
-            for delete_prev, delete_curr in zip(
-                delete_parameters, delete_parameters[1:]
-            )
-        ), "We cannot merge two delete parameters if their equality column names are different."
-        merge_delete_parameters = DeleteParameters.of(equality_column_names)
-        return merge_delete_parameters

deltacat/storage/model/partition_spec.py

@@ -1,71 +0,0 @@
-from __future__ import annotations
-from typing import List, Optional, Any
-from deltacat.storage.model.transform import Transform
-
-"""
-An ordered list of partition values determining the values of
-ordered transforms specified in the partition spec.
-"""
-PartitionValues = List[Any]
-
-
-class PartitionFilter(dict):
-    """
-    This class represents a filter for partitions.
-    It is used to filter partitions based on certain criteria.
-    """
-
-    @staticmethod
-    def of(
-        partition_values: Optional[PartitionValues] = None,
-    ) -> PartitionFilter:
-        """
-        Creates a new PartitionFilter instance with the specified partition key and value.
-        """
-        partition_filter = PartitionFilter()
-        partition_filter["partitionValues"] = partition_values
-        return partition_filter
-
-    @property
-    def partition_values(self) -> Optional[PartitionValues]:
-        return self.get("partitionValues")
-
-
-class PartitionSpec(dict):
-    """
-    This class determines how the underlying entities in the
-    hierarchy are partitioned. Stream partitions deltas and
-    delta partitions files.
-    """
-
-    @staticmethod
-    def of(ordered_transforms: List[Transform] = None) -> PartitionSpec:
-        partition_spec = PartitionSpec()
-        partition_spec.ordered_transforms = ordered_transforms
-        return partition_spec
-
-    @property
-    def ordered_transforms(self) -> List[Transform]:
-        return self.get("orderedTransforms")
-
-    @ordered_transforms.setter
-    def ordered_transforms(self, value: List[Transform]) -> None:
-        self["orderedTransforms"] = value
-
-
-class StreamPartitionSpec(PartitionSpec):
-    """
-    A class representing a stream partition specification.
-    A stream partitions deltas into multiple different Partition
-    """
-
-    pass
-
-
-class DeltaPartitionSpec(PartitionSpec):
-    """
-    A class representing delta partition specification.
-    The manifest entries in delta are partitioned based on this spec.
-    """
-
-    pass

deltacat/tests/compute/compactor/utils/test_round_completion_file.py

@@ -1,231 +0,0 @@
-import pytest
-import os
-from moto import mock_s3
-import boto3
-from boto3.resources.base import ServiceResource
-from deltacat.compute.compactor.utils.round_completion_file import (
-    read_round_completion_file,
-    write_round_completion_file,
-)
-from deltacat.tests.compute.test_util_common import get_test_partition_locator
-from deltacat.compute.compactor import RoundCompletionInfo
-
-RCF_BUCKET_NAME = "rcf-bucket"
-
-
-@pytest.fixture(autouse=True, scope="module")
-def mock_aws_credential():
-    os.environ["AWS_ACCESS_KEY_ID"] = "testing"
-    os.environ["AWS_SECRET_ACCESS_ID"] = "testing"
-    os.environ["AWS_SECURITY_TOKEN"] = "testing"
-    os.environ["AWS_SESSION_TOKEN"] = "testing"
-    os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
-    yield
-
-
-@pytest.fixture(autouse=True, scope="module")
-def s3_resource(mock_aws_credential):
-    with mock_s3():
-        yield boto3.resource("s3")
-
-
-@pytest.fixture(autouse=True, scope="function")
-def setup_compaction_artifacts_s3_bucket(s3_resource: ServiceResource):
-    s3_resource.create_bucket(
-        ACL="authenticated-read",
-        Bucket=RCF_BUCKET_NAME,
-    )
-    yield
-    s3_resource.Bucket(RCF_BUCKET_NAME).objects.all().delete()
-
-
-class TestReadWriteRoundCompletionFile:
-    def test_read_when_rcf_written_without_destination(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-        destination_locator = get_test_partition_locator("destination")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        rcf_url = write_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, None, expected_rcf
-        )
-
-        rcf = read_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator
-        )
-
-        assert (
-            rcf_url == "s3://rcf-bucket/f9829af39770d904dbb811bd8f4e886dd307f507.json"
-        )
-        assert rcf == expected_rcf
-
-    def test_read_when_rcf_written_with_destination(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-        destination_locator = get_test_partition_locator("destination")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        rcf_url = write_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator, expected_rcf
-        )
-
-        rcf = read_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator
-        )
-
-        assert (
-            rcf_url
-            == "s3://rcf-bucket/f9829af39770d904dbb811bd8f4e886dd307f507/e9939deadc091b3289a2eb0ca56b1ba86b9892f4.json"
-        )
-        assert rcf == expected_rcf
-
-    def test_read_without_destination_when_rcf_written_with_destination(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-        destination_locator = get_test_partition_locator("destination")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        write_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator, expected_rcf
-        )
-
-        rcf = read_round_completion_file(RCF_BUCKET_NAME, source_locator, None)
-
-        assert rcf is None
-
-    def test_read_without_destination_when_rcf_written_without_destination(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        write_round_completion_file(RCF_BUCKET_NAME, source_locator, None, expected_rcf)
-
-        rcf = read_round_completion_file(RCF_BUCKET_NAME, source_locator, None)
-
-        assert rcf == expected_rcf
-
-    def test_read_when_rcf_written_both_with_and_without_destination(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-        destination_locator = get_test_partition_locator("destination")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        expected_rcf_2 = RoundCompletionInfo.of(
-            high_watermark=1223,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=1233,
-        )
-
-        write_round_completion_file(RCF_BUCKET_NAME, source_locator, None, expected_rcf)
-
-        write_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator, expected_rcf_2
-        )
-
-        rcf = read_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator
-        )
-
-        assert rcf == expected_rcf_2
-
-    def test_read_when_none_destination_partition_id(self):
-
-        source_locator = get_test_partition_locator("source")
-        destination_locator = get_test_partition_locator(None)
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        write_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator, expected_rcf
-        )
-
-        rcf = read_round_completion_file(
-            RCF_BUCKET_NAME, source_locator, destination_locator
-        )
-
-        assert rcf == expected_rcf
-
-    def test_write_when_custom_url_is_passed(self):
-        """
-        This test case tests the backward compatibility by successfully
-        reading the previously written rcf.
-        """
-
-        source_locator = get_test_partition_locator("source")
-
-        expected_rcf = RoundCompletionInfo.of(
-            high_watermark=122,
-            compacted_delta_locator={},
-            compacted_pyarrow_write_result={},
-            sort_keys_bit_width=12,
-        )
-
-        completion_file_s3_url = f"s3://{RCF_BUCKET_NAME}/test.json"
-        rcf_url = write_round_completion_file(
-            RCF_BUCKET_NAME,
-            source_locator,
-            None,
-            expected_rcf,
-            completion_file_s3_url=completion_file_s3_url,
-        )
-
-        rcf = read_round_completion_file(RCF_BUCKET_NAME, source_locator, None)
-
-        assert rcf_url == completion_file_s3_url
-        assert rcf is None