deltacat 1.1.36__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

deltacat/catalog/default_catalog_impl/__init__.py

@@ -1,369 +0,0 @@
-from typing import Any, Dict, List, Optional, Set, Union, Tuple
-import pyarrow as pa
-import logging
-from deltacat.catalog.model.table_definition import TableDefinition
-from deltacat.storage.model.sort_key import SortKey
-from deltacat.storage.model.list_result import ListResult
-from deltacat.storage.model.namespace import Namespace
-from deltacat.storage.model.types import (
-    DistributedDataset,
-    LifecycleState,
-    LocalDataset,
-    LocalTable,
-    SchemaConsistencyType,
-)
-from deltacat.storage.model.partition import PartitionLocator, Partition
-from deltacat.storage.model.table_version import TableVersion
-from deltacat.compute.merge_on_read.model.merge_on_read_params import MergeOnReadParams
-from deltacat.storage.model.delta import DeltaType
-import deltacat.storage.interface as deltacat_storage
-from deltacat.types.media import ContentType, TableType, DistributedDatasetType
-from deltacat.types.tables import TableWriteMode
-from deltacat.compute.merge_on_read import MERGE_FUNC_BY_DISTRIBUTED_DATASET_TYPE
-from deltacat import logs
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-STORAGE = None
-
-
-# table functions
-def write_to_table(
-    data: Union[LocalTable, LocalDataset, DistributedDataset],  # type: ignore
-    table: str,
-    namespace: Optional[str] = None,
-    mode: TableWriteMode = TableWriteMode.AUTO,
-    content_type: ContentType = ContentType.PARQUET,
-    *args,
-    **kwargs,
-) -> None:
-    """Write local or distributed data to a table. Raises an error if the
-    table does not exist and the table write mode is not CREATE or AUTO.
-
-    When creating a table, all `create_table` parameters may be optionally
-    specified as additional keyword arguments. When appending to, or replacing,
-    an existing table, all `alter_table` parameters may be optionally specified
-    as additional keyword arguments."""
-    raise NotImplementedError("write_to_table not implemented")
-
-
-def read_table(
-    table: str,
-    namespace: Optional[str] = None,
-    table_version: Optional[str] = None,
-    table_type: Optional[TableType] = TableType.PYARROW,
-    distributed_dataset_type: Optional[
-        DistributedDatasetType
-    ] = DistributedDatasetType.RAY_DATASET,
-    partition_filter: Optional[List[Union[Partition, PartitionLocator]]] = None,
-    stream_position_range_inclusive: Optional[Tuple[int, int]] = None,
-    merge_on_read: Optional[bool] = False,
-    reader_kwargs: Optional[Dict[Any, Any]] = None,
-    deltacat_storage_kwargs: Optional[Dict[Any, Any]] = None,
-    *args,
-    **kwargs,
-) -> DistributedDataset:  # type: ignore
-    """Read a table into a distributed dataset."""
-
-    if reader_kwargs is None:
-        reader_kwargs = {}
-
-    if deltacat_storage_kwargs is None:
-        deltacat_storage_kwargs = {}
-
-    _validate_read_table_args(
-        namespace=namespace,
-        table_type=table_type,
-        distributed_dataset_type=distributed_dataset_type,
-        merge_on_read=merge_on_read,
-    )
-
-    table_version_obj = _get_latest_or_given_table_version(
-        namespace=namespace,
-        table_name=table,
-        table_version=table_version,
-        **deltacat_storage_kwargs,
-    )
-    table_version = table_version_obj.table_version
-
-    if (
-        table_version_obj.content_types is None
-        or len(table_version_obj.content_types) != 1
-    ):
-        raise ValueError(
-            "Expected exactly one content type but "
-            f"found {table_version_obj.content_types}."
-        )
-
-    logger.info(
-        f"Reading metadata for table={namespace}/{table}/{table_version} "
-        f"with partition_filters={partition_filter} and stream position"
-        f" range={stream_position_range_inclusive}"
-    )
-
-    if partition_filter is None:
-        logger.info(
-            f"Reading all partitions metadata in the table={table} "
-            "as partition_filter was None."
-        )
-        partition_filter = STORAGE.list_partitions(
-            table_name=table,
-            namespace=namespace,
-            table_version=table_version,
-            **deltacat_storage_kwargs,
-        ).all_items()
-
-    qualified_deltas = _get_deltas_from_partition_filter(
-        stream_position_range_inclusive=stream_position_range_inclusive,
-        partition_filter=partition_filter,
-        **deltacat_storage_kwargs,
-    )
-
-    logger.info(
-        f"Total qualified deltas={len(qualified_deltas)} "
-        f"from {len(partition_filter)} partitions."
-    )
-
-    merge_on_read_params = MergeOnReadParams.of(
-        {
-            "deltas": qualified_deltas,
-            "deltacat_storage": STORAGE,
-            "deltacat_storage_kwargs": deltacat_storage_kwargs,
-            "reader_kwargs": reader_kwargs,
-        }
-    )
-
-    return MERGE_FUNC_BY_DISTRIBUTED_DATASET_TYPE[distributed_dataset_type.value](
-        params=merge_on_read_params, **kwargs
-    )
-
-
-def alter_table(
-    table: str,
-    namespace: Optional[str] = None,
-    lifecycle_state: Optional[LifecycleState] = None,
-    schema_updates: Optional[Dict[str, Any]] = None,
-    partition_updates: Optional[Dict[str, Any]] = None,
-    primary_keys: Optional[Set[str]] = None,
-    sort_keys: Optional[List[SortKey]] = None,
-    description: Optional[str] = None,
-    properties: Optional[Dict[str, str]] = None,
-    *args,
-    **kwargs,
-) -> None:
-    """Alter table definition."""
-    raise NotImplementedError("alter_table not implemented")
-
-
-def create_table(
-    table: str,
-    namespace: Optional[str] = None,
-    lifecycle_state: Optional[LifecycleState] = None,
-    schema: Optional[Union[pa.Schema, str, bytes]] = None,
-    schema_consistency: Optional[Dict[str, SchemaConsistencyType]] = None,
-    partition_keys: Optional[List[Dict[str, Any]]] = None,
-    primary_keys: Optional[Set[str]] = None,
-    sort_keys: Optional[List[SortKey]] = None,
-    description: Optional[str] = None,
-    properties: Optional[Dict[str, str]] = None,
-    permissions: Optional[Dict[str, Any]] = None,
-    content_types: Optional[List[ContentType]] = None,
-    replace_existing_table: bool = False,
-    *args,
-    **kwargs,
-) -> TableDefinition:
-    """Create an empty table. Raises an error if the table already exists and
-    `replace_existing_table` is False."""
-    raise NotImplementedError("create_table not implemented")
-
-
-def drop_table(
-    table: str, namespace: Optional[str] = None, purge: bool = False, *args, **kwargs
-) -> None:
-    """Drop a table from the catalog and optionally purge it. Raises an error
-    if the table does not exist."""
-    raise NotImplementedError("drop_table not implemented")
-
-
-def refresh_table(table: str, namespace: Optional[str] = None, *args, **kwargs) -> None:
-    """Refresh metadata cached on the Ray cluster for the given table."""
-    raise NotImplementedError("refresh_table not implemented")
-
-
-def list_tables(
-    namespace: Optional[str] = None, *args, **kwargs
-) -> ListResult[TableDefinition]:
-    """List a page of table definitions. Raises an error if the given namespace
-    does not exist."""
-    raise NotImplementedError("list_tables not implemented")
-
-
-def get_table(
-    table: str, namespace: Optional[str] = None, *args, **kwargs
-) -> Optional[TableDefinition]:
-    """Get table definition metadata. Returns None if the given table does not
-    exist."""
-    raise NotImplementedError("get_table not implemented")
-
-
-def truncate_table(
-    table: str, namespace: Optional[str] = None, *args, **kwargs
-) -> None:
-    """Truncate table data. Raises an error if the table does not exist."""
-    raise NotImplementedError("truncate_table not implemented")
-
-
-def rename_table(
-    table: str, new_name: str, namespace: Optional[str] = None, *args, **kwargs
-) -> None:
-    """Rename a table."""
-    raise NotImplementedError("rename_table not implemented")
-
-
-def table_exists(table: str, namespace: Optional[str] = None, *args, **kwargs) -> bool:
-    """Returns True if the given table exists, False if not."""
-    raise NotImplementedError("table_exists not implemented")
-
-
-# namespace functions
-def list_namespaces(*args, **kwargs) -> ListResult[Namespace]:
-    """List a page of table namespaces."""
-    raise NotImplementedError("list_namespaces not implemented")
-
-
-def get_namespace(namespace: str, *args, **kwargs) -> Optional[Namespace]:
-    """Gets table namespace metadata for the specified table namespace. Returns
-    None if the given namespace does not exist."""
-    raise NotImplementedError("get_namespace not implemented")
-
-
-def namespace_exists(namespace: str, *args, **kwargs) -> bool:
-    """Returns True if the given table namespace exists, False if not."""
-    raise NotImplementedError("namespace_exists not implemented")
-
-
-def create_namespace(
-    namespace: str, permissions: Dict[str, Any], *args, **kwargs
-) -> Namespace:
-    """Creates a table namespace with the given name and permissions. Returns
-    the created namespace. Raises an error if the namespace already exists."""
-    raise NotImplementedError("create_namespace not implemented")
-
-
-def alter_namespace(
-    namespace: str,
-    permissions: Optional[Dict[str, Any]] = None,
-    new_namespace: Optional[str] = None,
-    *args,
-    **kwargs,
-) -> None:
-    """Alter table namespace definition."""
-    raise NotImplementedError("alter_namespace not implemented")
-
-
-def drop_namespace(namespace: str, purge: bool = False, *args, **kwargs) -> None:
-    """Drop the given namespace and all of its tables from the catalog,
-    optionally purging them."""
-    raise NotImplementedError("drop_namespace not implemented")
-
-
-def default_namespace() -> str:
-    """Returns the default namespace for the catalog."""
-    raise NotImplementedError("default_namespace not implemented")
-
-
-# catalog functions
-def initialize(ds: deltacat_storage, *args, **kwargs) -> None:
-    """Initializes the data catalog with the given arguments."""
-    global STORAGE
-    STORAGE = ds
-
-
-def _validate_read_table_args(
-    namespace: Optional[str] = None,
-    table_type: Optional[TableType] = None,
-    distributed_dataset_type: Optional[DistributedDatasetType] = None,
-    merge_on_read: Optional[bool] = None,
-):
-    if STORAGE is None:
-        raise ValueError(
-            "Catalog not initialized. Did you miss calling "
-            "initialize(ds=<deltacat_storage>)?"
-        )
-
-    if merge_on_read:
-        raise ValueError("Merge on read not supported currently.")
-
-    if table_type is not TableType.PYARROW:
-        raise ValueError("Only PYARROW table type is supported as of now")
-
-    if distributed_dataset_type is not DistributedDatasetType.DAFT:
-        raise ValueError("Only DAFT dataset type is supported as of now")
-
-    if namespace is None:
-        raise ValueError(
-            "namespace must be passed to uniquely identify a table in the catalog."
-        )
-
-
-def _get_latest_or_given_table_version(
-    namespace: str,
-    table_name: str,
-    table_version: Optional[str] = None,
-    *args,
-    **kwargs,
-) -> TableVersion:
-    table_version_obj = None
-    if table_version is None:
-        table_version_obj = STORAGE.get_latest_table_version(
-            namespace=namespace, table_name=table_name, *args, **kwargs
-        )
-        table_version = table_version_obj.table_version
-    else:
-        table_version_obj = STORAGE.get_table_version(
-            namespace=namespace,
-            table_name=table_name,
-            table_version=table_version,
-            *args,
-            **kwargs,
-        )
-
-    return table_version_obj
-
-
-def _get_deltas_from_partition_filter(
-    partition_filter: Optional[List[Union[Partition, PartitionLocator]]] = None,
-    stream_position_range_inclusive: Optional[Tuple[int, int]] = None,
-    *args,
-    **kwargs,
-):
-
-    result_deltas = []
-    start_stream_position, end_stream_position = stream_position_range_inclusive or (
-        None,
-        None,
-    )
-    for partition_like in partition_filter:
-        deltas = STORAGE.list_partition_deltas(
-            partition_like=partition_like,
-            ascending_order=True,
-            include_manifest=True,
-            start_stream_position=start_stream_position,
-            last_stream_position=end_stream_position,
-            *args,
-            **kwargs,
-        ).all_items()
-
-        for delta in deltas:
-            if (
-                start_stream_position is None
-                or delta.stream_position >= start_stream_position
-            ) and (
-                end_stream_position is None
-                or delta.stream_position <= end_stream_position
-            ):
-                if delta.type == DeltaType.DELETE:
-                    raise ValueError("DELETE type deltas are not supported")
-                result_deltas.append(delta)
-
-    return result_deltas

deltacat/io/dataset.py

@@ -1,73 +0,0 @@
-# Allow classes to use self-referencing Type hints in Python 3.7.
-from __future__ import annotations
-
-from typing import Any, Callable, Dict, Optional, TypeVar, Union, cast
-
-import pyarrow as pa
-import s3fs
-from ray.data import Dataset
-
-T = TypeVar("T")
-
-
-class DeltacatDataset(Dataset[T]):
-    @staticmethod
-    def from_dataset(dataset: Dataset[T]) -> DeltacatDataset[T]:
-        # cast to DeltacatDataset in-place since it only adds new methods
-        dataset.__class__ = DeltacatDataset
-        return cast(DeltacatDataset[T], dataset)
-
-    def write_redshift(
-        self,
-        path: str,
-        *,
-        filesystem: Optional[Union[pa.fs.FileSystem, s3fs.S3FileSystem]] = None,
-        try_create_dir: bool = True,
-        arrow_open_stream_args: Optional[Dict[str, Any]] = None,
-        arrow_parquet_args_fn: Callable[[], Dict[str, Any]] = lambda: {},
-        **arrow_parquet_args,
-    ) -> None:
-        """Writes the dataset to Parquet files and commits a Redshift manifest
-        back to S3 indexing the files written. The output can be loaded into
-        Redshift by providing it to the Redshift COPY command, or via AWS Data
-        Wrangler's `wr.redshift.copy_from_files()` API.
-
-        This is only supported for datasets convertible to Arrow records.
-        To control the number of files, use ``.repartition()``.
-
-        Unless a custom block path provider is given, the format of the output
-        files will be {uuid}_{block_idx}.parquet, where ``uuid`` is an unique
-        id for the dataset.
-
-        The Redshift manifest will be written to ``f"{path}/manifest``
-
-        Examples:
-            >>> ds.write_redshift("s3://bucket/path")
-
-        Time complexity: O(dataset size / parallelism)
-
-        Args:
-            path: The path to the destination root directory where Parquet
-                files and the Redshift manifest will be written to.
-            filesystem: The filesystem implementation to write to. This should
-                be either PyArrow's S3FileSystem or s3fs.
-            try_create_dir: Try to create all directories in destination path
-                if True. Does nothing if all directories already exist.
-            arrow_open_stream_args: kwargs passed to
-                pyarrow.fs.FileSystem.open_output_stream
-            filename_provider: FilenameProvider implementation
-                to write each dataset block to a custom output path.
-            arrow_parquet_args_fn: Callable that returns a dictionary of write
-                arguments to use when writing each block to a file. Overrides
-                any duplicate keys from arrow_parquet_args. This should be used
-                instead of arrow_parquet_args if any of your write arguments
-                cannot be pickled, or if you'd like to lazily resolve the write
-                arguments for each dataset block.
-            arrow_parquet_args: Options to pass to
-                pyarrow.parquet.write_table(), which is used to write out each
-                block to a file.
-        """
-        raise NotImplementedError(
-            "Writing to Redshift is not yet supported. "
-            "Please use DeltacatDataset.write_parquet() instead."
-        )

deltacat/io/read_api.py

@@ -1,143 +0,0 @@
-from typing import Any, Callable, Dict, List, Optional, Union
-
-import pyarrow as pa
-import s3fs
-from ray.data import read_datasource
-from ray.data._internal.arrow_block import ArrowRow
-
-from deltacat import ContentType
-from deltacat.io.aws.redshift.redshift_datasource import (
-    HivePartitionParser,
-    RedshiftDatasource,
-    RedshiftUnloadTextArgs,
-    S3PathType,
-)
-from deltacat.io.dataset import DeltacatDataset
-from deltacat.utils.common import ReadKwargsProvider
-
-
-def read_redshift(
-    paths: Union[str, List[str]],
-    *,
-    path_type: S3PathType = S3PathType.MANIFEST,
-    filesystem: Optional[Union[pa.fs.S3FileSystem, s3fs.S3FileSystem]] = None,
-    columns: Optional[List[str]] = None,
-    schema: Optional[pa.Schema] = None,
-    unload_text_args: RedshiftUnloadTextArgs = RedshiftUnloadTextArgs(),
-    partitioning: HivePartitionParser = None,
-    content_type_provider: Callable[[str], ContentType] = lambda p: ContentType.PARQUET
-    if p.endswith(".parquet")
-    else ContentType.CSV,
-    parallelism: int = 200,
-    ray_remote_args: Dict[str, Any] = None,
-    arrow_open_stream_args: Optional[Dict[str, Any]] = None,
-    pa_read_func_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    **kwargs,
-) -> DeltacatDataset[ArrowRow]:
-    """Reads Redshift UNLOAD results from either S3 Parquet or delimited text
-    files into a Ray Dataset.
-
-    Examples:
-        >>> # Read all files contained in a Redshift Manifest:
-        >>> import deltacat as dc
-        >>> dc.io.read_redshift("/bucket/dir/manifest")
-
-        >>> # Read all files matching the given key prefix. If this prefix
-        >>> # refers to multiple files, like s3://bucket/data.parquet,
-        >>> # s3://bucket/data.1.csv, etc. then all will be read. The dataset
-        >>> # schema will be inferred from the first parquet file and used for
-        >>> # explicit type conversion of all CSV files:
-        >>> dc.io.read_redshift(
-        >>>     "s3://bucket/data.txt",
-        >>>     path_type=S3PathType.PREFIX)
-
-        >>> # Read all files matching the given key prefix. If this prefix
-        >>> # refers to multiple files or folders, like s3://bucket/dir/,
-        >>> # s3://bucket/dir1/, s3://bucket/dir.txt, s3://bucket/dir.txt.1,
-        >>> # then all files and subfolder contents will be read.
-        >>> dc.io.read_redshift(
-        >>>     "/bucket/dir",
-        >>>     path_type=S3PathType.PREFIX)
-
-        >>> # Read multiple files and folders:
-        >>> dc.io.read_redshift(
-        >>>     ["/bucket/file1", "/bucket/folder1/"],
-        >>>     path_type=S3PathType.FILES_AND_FOLDERS)
-
-        >>> # Read multiple Parquet and CSV files. The dataset schema will be
-        >>> # inferred from the first parquet file and used for explicit type
-        >>> # conversion of all CSV files:
-        >>> dc.io.read_redshift(
-        >>>     ["/bucket/file.parquet", "/bucket/file.csv"],
-        >>>     path_type=S3PathType.FILES_AND_FOLDERS)
-
-    Args:
-        paths: Paths to S3 files and folders to read. If `path_type` is
-            `MANIFEST` then this must be an S3 Redshift Manifest JSON file. If
-            `path_type` is `PREFIX` then this must be a valid S3 key prefix.
-            All files matching the key prefix, including files in matching
-            subdirectories, will be read. Unless custom
-            `content_type_extensions` are specified, file content types will be
-            inferred by file extension with ".parquet" used for Parquet files,
-            and all others assumed to be delimited text (e.g. CSV). It's
-            recommended to specify the path to a manifest unloaded with the
-            VERBOSE option whenever possible to improve the correctness and
-            performance of Dataset reads, compute operations, and writes.
-            `FILES_AND_FOLDERS` is not recommended when reading thousands of
-            files due to its relatively high-latency.
-        path_type: Determines how the `paths` parameter is interpreted.
-        filesystem: The filesystem implementation to read from. This should be
-            either PyArrow's S3FileSystem or s3fs.
-        columns: A list of column names to read. Reads all columns if None or
-            empty.
-        schema: PyArrow schema used to determine delimited text column
-            names and types. If not specified and both Parquet and delimited
-            text files are read as input, then the first Parquet file schema
-            discovered is used instead.
-        unload_text_args: Arguments used when running Redshift `UNLOAD` to
-            text file formats (e.g. CSV). These arguments ensure that all input
-            text files will be correctly parsed. If not specified, then all
-            text files read are assumed to use Redshift UNLOAD's default
-            pipe-delimited text format.
-        partition_base_dir: Base directory to start searching for partitions
-            (exclusive). File paths outside of this directory will not be parsed
-            for partitions and automatically added to the dataset without passing
-            through any partition filter. Specify `None` or an empty string to
-            search for partitions in all file path directories.
-        partition_filter_fn: Callback used to filter `PARTITION` columns. Receives a
-            dictionary mapping partition keys to values as input, returns `True` to
-            read a partition, and `False` to skip it. Each partition key and value
-            is a string parsed directly from an S3 key using hive-style
-            partition directory names of the form "{key}={value}". For example:
-            ``lambda x:
-            True if x["month"] == "January" and x["year"] == "2022" else False``
-        content_type_provider: Takes a file path as input and returns the file
-            content type as output.
-        parallelism: The requested parallelism of the read. Parallelism may be
-            limited by the number of files of the dataset.
-        ray_remote_args: kwargs passed to `ray.remote` in the read tasks.
-        arrow_open_stream_args: kwargs passed to to
-            `pa.fs.open_input_stream()`.
-        pa_read_func_kwargs_provider: Callback that takes a `ContentType` value
-            string as input, and provides read options to pass to either
-            `pa.csv.open_csv()` or `pa.parquet.read_table()` as output.
-    Returns:
-        Dataset holding Arrow records read from the specified paths.
-    """
-    dataset = read_datasource(
-        RedshiftDatasource(),
-        parallelism=parallelism,
-        paths=paths,
-        content_type_provider=content_type_provider,
-        path_type=path_type,
-        filesystem=filesystem,
-        columns=columns,
-        schema=schema,
-        unload_args=unload_text_args,
-        partitioning=partitioning,
-        ray_remote_args=ray_remote_args,
-        open_stream_args=arrow_open_stream_args,
-        read_kwargs_provider=pa_read_func_kwargs_provider,
-        **kwargs,
-    )
-    return DeltacatDataset.from_dataset(dataset)

deltacat/storage/model/delete_parameters.py

@@ -1,40 +0,0 @@
-# Allow classes to use self-referencing Type hints in Python 3.7.
-from __future__ import annotations
-
-from typing import List, Optional
-
-
-class DeleteParameters(dict):
-    """
-    Contains all parameters required to support DELETEs
-    equality_column_names: List of column names that would be used to determine row equality for equality deletes.  Relevant only to equality deletes
-    """
-
-    @staticmethod
-    def of(
-        equality_column_names: Optional[List[str]] = None,
-    ) -> DeleteParameters:
-        delete_parameters = DeleteParameters()
-        if equality_column_names is not None:
-            delete_parameters["equality_column_names"] = equality_column_names
-        return delete_parameters
-
-    @property
-    def equality_column_names(self) -> Optional[List[str]]:
-        return self.get("equality_column_names")
-
-    @staticmethod
-    def merge_delete_parameters(
-        delete_parameters: List[DeleteParameters],
-    ) -> Optional[DeleteParameters]:
-        if len(delete_parameters) < 2:
-            return delete_parameters
-        equality_column_names = delete_parameters[0].equality_column_names
-        assert all(
-            delete_prev.equality_column_names == delete_curr.equality_column_names
-            for delete_prev, delete_curr in zip(
-                delete_parameters, delete_parameters[1:]
-            )
-        ), "We cannot merge two delete parameters if their equality column names are different."
-        merge_delete_parameters = DeleteParameters.of(equality_column_names)
-        return merge_delete_parameters

deltacat/storage/model/partition_spec.py

@@ -1,71 +0,0 @@
-from __future__ import annotations
-from typing import List, Optional, Any
-from deltacat.storage.model.transform import Transform
-
-"""
-An ordered list of partition values determining the values of
-ordered transforms specified in the partition spec.
-"""
-PartitionValues = List[Any]
-
-
-class PartitionFilter(dict):
-    """
-    This class represents a filter for partitions.
-    It is used to filter partitions based on certain criteria.
-    """
-
-    @staticmethod
-    def of(
-        partition_values: Optional[PartitionValues] = None,
-    ) -> PartitionFilter:
-        """
-        Creates a new PartitionFilter instance with the specified partition key and value.
-        """
-        partition_filter = PartitionFilter()
-        partition_filter["partitionValues"] = partition_values
-        return partition_filter
-
-    @property
-    def partition_values(self) -> Optional[PartitionValues]:
-        return self.get("partitionValues")
-
-
-class PartitionSpec(dict):
-    """
-    This class determines how the underlying entities in the
-    hierarchy are partitioned. Stream partitions deltas and
-    delta partitions files.
-    """
-
-    @staticmethod
-    def of(ordered_transforms: List[Transform] = None) -> PartitionSpec:
-        partition_spec = PartitionSpec()
-        partition_spec.ordered_transforms = ordered_transforms
-        return partition_spec
-
-    @property
-    def ordered_transforms(self) -> List[Transform]:
-        return self.get("orderedTransforms")
-
-    @ordered_transforms.setter
-    def ordered_transforms(self, value: List[Transform]) -> None:
-        self["orderedTransforms"] = value
-
-
-class StreamPartitionSpec(PartitionSpec):
-    """
-    A class representing a stream partition specification.
-    A stream partitions deltas into multiple different Partition
-    """
-
-    pass
-
-
-class DeltaPartitionSpec(PartitionSpec):
-    """
-    A class representing delta partition specification.
-    The manifest entries in delta are partitioned based on this spec.
-    """
-
-    pass