deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/experimental/storage/iceberg/visitor.py

@@ -0,0 +1,119 @@
+import logging
+from typing import Any
+
+import pyarrow
+from deltacat.storage.model.scan.push_down import PartitionFilter
+
+import deltacat.logs as logs
+from deltacat.storage.model.expression import Reference, Literal
+from deltacat.storage.model.expression.visitor import ExpressionVisitor
+from pyiceberg.expressions import (
+    And,
+    Or,
+    Not,
+    EqualTo,
+    NotEqualTo,
+    GreaterThan,
+    GreaterThanOrEqual,
+    LessThan,
+    LessThanOrEqual,
+    IsNull,
+    In,
+)
+
+# Initialize DeltaCAT logger
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+class IcebergExpressionVisitor(ExpressionVisitor[None, Any]):
+    """
+    Visitor that translates DeltaCAT expressions to PyIceberg expressions.
+    """
+
+    def visit(self, expr, context=None):
+        # Handle PartitionFilter by extracting and visiting the inner expression
+        if isinstance(expr, PartitionFilter):
+            return self.visit(expr.expr, context)
+        # Handle all other expressions using the parent's visit method
+        return super().visit(expr, context)
+
+    def visit_reference(self, expr: Reference, context=None) -> str:
+        return expr.field
+
+    def visit_literal(self, expr: Literal, context=None) -> Any:
+        # Convert PyArrow scalar to Python native type
+        return (
+            expr.value.as_py() if isinstance(expr.value, pyarrow.Scalar) else expr.value
+        )
+
+    def visit_and(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return And(left, right)
+
+    def visit_or(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return Or(left, right)
+
+    def visit_not(self, expr, context=None):
+        operand = self.visit(expr.operand, context)
+        return Not(operand)
+
+    def visit_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return EqualTo(left, right)
+
+    def visit_not_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return NotEqualTo(left, right)
+
+    def visit_greater_than(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return GreaterThan(left, right)
+
+    def visit_greater_than_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return GreaterThanOrEqual(left, right)
+
+    def visit_less_than(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return LessThan(left, right)
+
+    def visit_less_than_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return LessThanOrEqual(left, right)
+
+    def visit_is_null(self, expr, context=None):
+        operand = self.visit(expr.operand, context)
+        return IsNull(operand)
+
+    def visit_in(self, expr, context=None):
+        value = self.visit(expr.value, context)
+        values = [self.visit(v, context) for v in expr.values]
+        return In(value, values)
+
+    def visit_between(self, expr, context=None):
+        value = self.visit(expr.value, context)
+        lower = self.visit(expr.lower, context)
+        upper = self.visit(expr.upper, context)
+        return And(GreaterThanOrEqual(value, lower), LessThanOrEqual(value, upper))
+
+    # PyIceberg does not have a direct equivalent of LIKE
+    def visit_like(self, expr, context=None):
+        value = self.visit(expr.value, context)
+        pattern = self.visit(expr.pattern, context)
+        logger.warning(
+            f"LIKE operation is not supported in PyIceberg. Ignoring LIKE filter: {value} LIKE '{pattern}'. "
+            "This may result in more data being returned than expected."
+        )
+        # Return None or a default expression that won't filter anything
+        return (
+            None  # or return NotEqualTo(value, None)  # matches everything except NULL
+        )

deltacat/experimental/storage/rivulet/__init__.py

@@ -0,0 +1,11 @@
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.schema.schema import Field
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.schema.schema import Datatype
+
+__all__ = [
+    "Schema",
+    "Field",
+    "Dataset",
+    "Datatype",
+]

deltacat/{storage → experimental/storage}/rivulet/arrow/serializer.py

@@ -2,10 +2,13 @@ from abc import ABC, abstractmethod
 from typing import Iterator, List, Any
 import pyarrow as pa
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.serializer import DataSerializer, MEMTABLE_DATA
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.serializer import (
+    DataSerializer,
+    MEMTABLE_DATA,
+)
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 
 class ArrowSerializer(DataSerializer, ABC):

deltacat/{storage → experimental/storage}/rivulet/dataset.py

@@ -24,19 +24,23 @@ from deltacat.storage.model.shard import Shard, ShardingStrategy
 from deltacat.storage.model.stream import Stream, StreamLocator
 from deltacat.storage.model.transaction import TransactionOperationList
 from deltacat.storage.model.types import CommitState, StreamFormat
-from deltacat.storage.rivulet.fs.file_store import FileStore
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.reader.dataset_metastore import DatasetMetastore
-from deltacat.storage.rivulet import Schema, Field
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+from deltacat.experimental.storage.rivulet import Schema, Field
 from deltacat.utils.export import export_dataset
 from .schema.schema import Datatype
 
-from deltacat.storage.rivulet.reader.data_scan import DataScan
-from deltacat.storage.rivulet.reader.dataset_reader import DatasetReader
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
+from deltacat.experimental.storage.rivulet.reader.data_scan import DataScan
+from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
 
-from deltacat.storage.rivulet.writer.dataset_writer import DatasetWriter
-from deltacat.storage.rivulet.writer.memtable_dataset_writer import (
+from deltacat.experimental.storage.rivulet.writer.dataset_writer import DatasetWriter
+from deltacat.experimental.storage.rivulet.writer.memtable_dataset_writer import (
     MemtableDatasetWriter,
 )
 
@@ -48,7 +52,6 @@ from deltacat.storage import (
     TableVersion,
     TableVersionLocator,
     Transaction,
-    TransactionType,
     TransactionOperation,
     TransactionOperationType,
 )
@@ -295,7 +298,6 @@ class Dataset:
                     partition_values=DEFAULT_PARTITION_VALUES,
                     partition_id=self._partition_id,
                 ),
-                schema=None,
                 content_types=None,
             ),
         ]
@@ -308,7 +310,6 @@ class Dataset:
         ]
 
         transaction = Transaction.of(
-            txn_type=TransactionType.APPEND,
             txn_operations=TransactionOperationList.of(txn_operations),
         )
 

deltacat/{storage → experimental/storage}/rivulet/dataset_executor.py

@@ -2,13 +2,16 @@ from __future__ import annotations
 
 from typing import List, Callable, Any
 
-from deltacat.storage.rivulet.field_group import FieldGroup
-from deltacat.storage.rivulet.mvp.Table import MvpTable
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.reader.data_scan import DataScan
-from deltacat.storage.rivulet.reader.dataset_metastore import DatasetMetastore
-from deltacat.storage.rivulet.reader.dataset_reader import DatasetReader
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
+from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.data_scan import DataScan
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
 
 
 class DatasetExecutor:
@@ -22,12 +25,10 @@ class DatasetExecutor:
 
     def __init__(
         self,
-        field_groups: List[FieldGroup],
         schema: Schema,
         metastore: DatasetMetastore,
     ):
         self.effective_schema: Schema = schema.__deepcopy__()
-        self.field_groups = field_groups
         self.output: MvpTable | None = None
         self._metastore = metastore
 
@@ -64,18 +65,9 @@ class DatasetExecutor:
 
         TODO for now this is doing dumb in-memory implementation and later this is going to be replaced by rust library
         """
-        if len(self.field_groups) == 1:
-            return self._read_as_mvp_table(schema, self.field_groups[0])
-        else:
-            ds1 = self._read_as_mvp_table(schema, self.field_groups[0])
-            ds2 = self._read_as_mvp_table(schema, self.field_groups[1])
-            merged = MvpTable.merge(ds1, ds2, schema.primary_key.name)
-            for i in range(2, len(self.field_groups)):
-                ds_i = self._read_as_mvp_table(schema, self.field_groups[i])
-                merged = MvpTable.merge(merged, ds_i, schema.primary_key.name)
-            return merged
+        return self._read_as_mvp_table(schema)
 
-    def _read_as_mvp_table(self, schema: Schema, field_group: FieldGroup):
+    def _read_as_mvp_table(self, schema: Schema):
         data = list(
             DataScan(
                 schema, QueryExpression(), DatasetReader(self._metastore)

deltacat/experimental/storage/rivulet/feather/__init__.py

@@ -0,0 +1,7 @@
+# TODO later on this will be moved to a dedicated package
+from deltacat.experimental.storage.rivulet.feather.file_reader import FeatherFileReader
+from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
+    FileReaderRegistrar,
+)
+
+FileReaderRegistrar.register_reader("feather", FeatherFileReader)

deltacat/{storage → experimental/storage}/rivulet/feather/file_reader.py

@@ -5,15 +5,17 @@ from typing import Optional
 import pyarrow.ipc
 from pyarrow import RecordBatch, RecordBatchFileReader
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.reader.data_reader import (
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
     RowAndKey,
     FileReader,
     FILE_FORMAT,
 )
-from deltacat.storage.rivulet.reader.pyarrow_data_reader import RecordBatchRowIndex
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
+    RecordBatchRowIndex,
+)
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 
 
 class FeatherFileReader(FileReader[RecordBatchRowIndex]):

deltacat/{storage → experimental/storage}/rivulet/feather/serializer.py

@@ -3,10 +3,10 @@ from typing import List
 import pyarrow as pa
 from pyarrow import feather
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.arrow.serializer import ArrowSerializer
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.arrow.serializer import ArrowSerializer
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 
 class FeatherDataSerializer(ArrowSerializer):

deltacat/{storage → experimental/storage}/rivulet/fs/file_provider.py

@@ -3,9 +3,9 @@ import time
 from typing import List, Generator
 
 from deltacat.storage.model.partition import PartitionLocator
-from deltacat.storage.rivulet.fs.file_store import FileStore
-from deltacat.storage.rivulet.fs.input_file import InputFile
-from deltacat.storage.rivulet.fs.output_file import OutputFile
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
 from deltacat.utils.metafile_locator import _find_partition_path
 
 

deltacat/{storage → experimental/storage}/rivulet/fs/file_store.py

@@ -4,8 +4,8 @@ from pyarrow.fs import FileSystem, FileType, FileSelector
 # TODO(deltacat): Rely on deltacat implementation to resolve path and filesystem.
 from ray.data.datasource.path_util import _resolve_paths_and_filesystem
 
-from deltacat.storage.rivulet.fs.input_file import FSInputFile
-from deltacat.storage.rivulet.fs.output_file import FSOutputFile
+from deltacat.experimental.storage.rivulet.fs.input_file import FSInputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import FSOutputFile
 
 
 class FileStore:

deltacat/{storage → experimental/storage}/rivulet/fs/output_file.py

@@ -5,7 +5,7 @@ from typing import Protocol
 
 from pyarrow.fs import FileSystem, FileType
 
-from deltacat.storage.rivulet.fs.input_file import FSInputFile, InputFile
+from deltacat.experimental.storage.rivulet.fs.input_file import FSInputFile, InputFile
 
 
 class OutputStream(Protocol):  # pragma: no cover

deltacat/{storage → experimental/storage}/rivulet/logical_plan.py

@@ -1,9 +1,9 @@
 from dataclasses import dataclass
 from typing import List, Callable, Any, Protocol
 
-from deltacat.storage.rivulet.dataset_executor import DatasetExecutor
-from deltacat.storage.rivulet.mvp.Table import MvpTable
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.dataset_executor import DatasetExecutor
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
 
 
 class DatasetOperation(Protocol):
@@ -99,7 +99,7 @@ class LogicalPlan:
         self.operations.append(CollectOperation())
         return self
 
-    def execute(self, executor: DatasetExecutor) -> "MvpTable":
+    def execute(self, executor: DatasetExecutor) -> MvpTable:
         for operation in self.operations:
             operation.visit(executor)
         return executor.output

deltacat/{storage → experimental/storage}/rivulet/metastore/delta.py

@@ -10,7 +10,6 @@ from deltacat.storage import (
     Delta,
     DeltaType,
     Transaction,
-    TransactionType,
     TransactionOperation,
     TransactionOperationType,
 )
@@ -19,7 +18,7 @@ from deltacat.storage.model.partition import PartitionLocator
 from deltacat.storage.model.transaction import TransactionOperationList
 
 from deltacat.storage.model.types import StreamFormat
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet import Schema
 
 StreamPosition = int
 """The stream position for creating a consistent ordering of manifests."""
@@ -169,7 +168,6 @@ class DeltacatManifestIO(ManifestIO):
         delta["level"] = level
 
         tx_results = Transaction.of(
-            txn_type=TransactionType.APPEND,
             txn_operations=TransactionOperationList.of(
                 [
                     TransactionOperation.of(

deltacat/{storage → experimental/storage}/rivulet/metastore/json_sst.py

@@ -4,9 +4,9 @@ import json
 from itertools import zip_longest
 from typing import List
 
-from deltacat.storage.rivulet.fs.input_file import InputFile
-from deltacat.storage.rivulet.fs.output_file import OutputFile
-from deltacat.storage.rivulet.metastore.sst import (
+from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
+from deltacat.experimental.storage.rivulet.metastore.sst import (
     SSTWriter,
     SSTableRow,
     SSTReader,

deltacat/{storage → experimental/storage}/rivulet/metastore/sst.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from typing import Protocol, Any, List
 
-from deltacat.storage.rivulet.fs.input_file import InputFile
-from deltacat.storage.rivulet.fs.output_file import OutputFile
+from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
 
 
 @dataclass(frozen=True)

deltacat/{storage → experimental/storage}/rivulet/metastore/sst_interval_tree.py

@@ -8,9 +8,9 @@ from typing import Any, Dict, Set, List, FrozenSet, Iterable, TypeVar, NamedTupl
 
 from intervaltree import Interval, IntervalTree
 
-from deltacat.storage.rivulet.metastore.delta import DeltaContext
-from deltacat.storage.rivulet.metastore.sst import SSTable, SSTableRow
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTable, SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
 
 T = TypeVar("T")
 

deltacat/experimental/storage/rivulet/parquet/__init__.py

@@ -0,0 +1,7 @@
+# TODO later on this will be moved to a dedicated package
+from deltacat.experimental.storage.rivulet.parquet.file_reader import ParquetFileReader
+from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
+    FileReaderRegistrar,
+)
+
+FileReaderRegistrar.register_reader("parquet", ParquetFileReader)

deltacat/{storage → experimental/storage}/rivulet/parquet/file_reader.py

@@ -4,15 +4,17 @@ from typing import Optional
 
 from pyarrow import RecordBatch
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.reader.data_reader import (
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
     RowAndKey,
     FileReader,
     FILE_FORMAT,
 )
-from deltacat.storage.rivulet.reader.pyarrow_data_reader import RecordBatchRowIndex
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
+    RecordBatchRowIndex,
+)
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 import pyarrow.parquet as pq
 import pyarrow as pa
 

deltacat/{storage → experimental/storage}/rivulet/parquet/serializer.py

@@ -3,11 +3,11 @@ from typing import List, Any
 import pyarrow as pa
 from pyarrow.parquet import FileMetaData
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.arrow.serializer import ArrowSerializer
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.arrow.serializer import ArrowSerializer
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 
 class ParquetDataSerializer(ArrowSerializer):

deltacat/{storage → experimental/storage}/rivulet/reader/block_scanner.py

@@ -15,19 +15,30 @@ from typing import (
     AbstractSet,
 )
 
-from deltacat.storage.rivulet.metastore.delta import DeltaContext
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.metastore.sst_interval_tree import (
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
     OrderedBlockGroups,
     BlockGroup,
     Block,
 )
-from deltacat.storage.rivulet.reader.data_reader import RowAndKey, FileReader
-from deltacat.storage.rivulet.reader.dataset_metastore import DatasetMetastore
-from deltacat.storage.rivulet.reader.pyarrow_data_reader import ArrowDataReader
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet.reader.reader_type_registrar import FileReaderRegistrar
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
+    RowAndKey,
+    FileReader,
+)
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
+    ArrowDataReader,
+)
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
+    FileReaderRegistrar,
+)
+from deltacat.experimental.storage.rivulet import Schema
 from deltacat import logs
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))

deltacat/{storage → experimental/storage}/rivulet/reader/data_reader.py

@@ -13,9 +13,9 @@ from typing import (
     Optional,
 )
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 
 FILE_FORMAT = TypeVar("FILE_FORMAT")
 MEMORY_FORMAT = TypeVar("MEMORY_FORMAT")

deltacat/{storage → experimental/storage}/rivulet/reader/data_scan.py

@@ -3,9 +3,11 @@ from typing import Generator, Dict, Optional
 import pyarrow as pa
 
 from deltacat.storage.model.shard import Shard
-from deltacat.storage.rivulet.reader.dataset_reader import DatasetReader
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet import Schema
 
 
 class DataScan:

deltacat/{storage → experimental/storage}/rivulet/reader/dataset_metastore.py

@@ -5,18 +5,19 @@ from typing import Generator, Optional
 import pyarrow
 import pyarrow.fs
 
+from deltacat.constants import REV_DIR_NAME
 from deltacat.storage import Delta
 from deltacat.storage.model.partition import PartitionLocator
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 from deltacat.utils.filesystem import resolve_path_and_filesystem
-from deltacat.storage.rivulet.metastore.json_sst import JsonSstReader
-from deltacat.storage.rivulet.metastore.delta import (
+from deltacat.experimental.storage.rivulet.metastore.json_sst import JsonSstReader
+from deltacat.experimental.storage.rivulet.metastore.delta import (
     ManifestIO,
     DeltaContext,
     RivuletDelta,
     DeltacatManifestIO,
 )
-from deltacat.storage.rivulet.metastore.sst import SSTReader, SSTable
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTReader, SSTable
 from deltacat.utils.metafile_locator import _find_table_path
 from deltacat import logs
 
@@ -83,7 +84,7 @@ class DatasetMetastore:
         param: filesystem: The filesystem to search for the revisions.
         returns: The latest revision as a RivuletDelta.
         """
-        rev_directory = posixpath.join(delta_dir, "rev")
+        rev_directory = posixpath.join(delta_dir, REV_DIR_NAME)
         revisions = filesystem.get_file_info(
             pyarrow.fs.FileSelector(rev_directory, allow_not_found=True)
         )
@@ -128,7 +129,7 @@ class DatasetMetastore:
             return
 
         # Locate "rev" directory inside the partition
-        rev_directory = posixpath.join(partition_path, "rev")
+        rev_directory = posixpath.join(partition_path, REV_DIR_NAME)
         rev_info = filesystem.get_file_info(rev_directory)
 
         if rev_info.type != pyarrow.fs.FileType.Directory:

deltacat/{storage → experimental/storage}/rivulet/reader/dataset_reader.py

@@ -2,18 +2,20 @@ import logging
 from typing import Generator, Optional, Set, Type, TypeVar, Any
 
 from deltacat.storage.model.shard import Shard
-from deltacat.storage.rivulet.metastore.sst import SSTableRow, SSTable
-from deltacat.storage.rivulet.metastore.sst_interval_tree import (
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow, SSTable
+from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
     BlockIntervalTree,
     OrderedBlockGroups,
 )
-from deltacat.storage.rivulet.reader.block_scanner import BlockScanner
-from deltacat.storage.rivulet.reader.dataset_metastore import (
+from deltacat.experimental.storage.rivulet.reader.block_scanner import BlockScanner
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
     DatasetMetastore,
     ManifestAccessor,
 )
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet import Schema
 
 # The type of data returned to reader
 T = TypeVar("T")

deltacat/{storage → experimental/storage}/rivulet/reader/pyarrow_data_reader.py

@@ -4,7 +4,10 @@ from typing import Generator, Dict, Type, NamedTuple, List
 
 from pyarrow import RecordBatch
 
-from deltacat.storage.rivulet.reader.data_reader import DataReader, MEMORY_FORMAT
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
+    DataReader,
+    MEMORY_FORMAT,
+)
 import pyarrow as pa