deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b11__py3-none-any.whl

deltacat/compute/jobs/client.py

@@ -70,6 +70,7 @@ def _get_head_node_ip(cluster_cfg: str) -> str:
         check=True,
     )
     # the head node IP should be the last line printed to stdout
+    # TODO(pdames): add IPv6 support
     head_node_ip = proc.stdout.splitlines()[-1]
     if not re.match(
         r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$",
@@ -322,12 +323,12 @@ class DeltaCatJobClient(JobSubmissionClient):
 
 def local_job_client(*args, **kwargs) -> DeltaCatJobClient:
     """
-    Create a Ray Job Client that can be used to submit jobs to a local Ray
+    Create a DeltaCAT Job Client that can be used to submit jobs to a local Ray
     cluster. Initializes Ray if it's not already running.
 
     Args:
-        *args: Positional arguments to pass to `ray.init()`.
-        **kwargs: Keyword arguments to pass to `ray.init()`.
+        *args: Positional arguments to pass to `deltacat.init()`.
+        **kwargs: Keyword arguments to pass to `deltacat.init()`.
     Returns:
         DeltaCatJobClient: A client instance that can be used to submit and
             manage local Ray jobs.
@@ -338,7 +339,7 @@ def local_job_client(*args, **kwargs) -> DeltaCatJobClient:
     if not dc.is_initialized():
         context = dc.init(*args, **kwargs)
     else:
-        context = dc.init(ray_init_args={"ignore_reinit_error": True})
+        context = dc.init()
     if context.dashboard_url:
         head_node_ip, port = context.dashboard_url.split(":")
     else:
@@ -367,7 +368,8 @@ def job_client(
     port: Union[str, int] = "8265",
 ) -> DeltaCatJobClient:
     """
-    Create a DeltaCAT Job Client that can be used to submit jobs to a remote Ray cluster.
+    Create a DeltaCAT Job Client that can be used to submit jobs to a remote
+    Ray cluster.
 
     Args:
         cluster_cfg_file_path: Path to the Ray Cluster Launcher

deltacat/constants.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 
 from deltacat.utils.common import env_string, env_bool
-import os
 
 # Environment variables
 DELTACAT_SYS_LOG_LEVEL = env_string("DELTACAT_SYS_LOG_LEVEL", "DEBUG")
@@ -40,7 +39,7 @@ DELTACAT_LOGGER_USE_SINGLE_HANDLER = env_bool(
 )
 DELTACAT_ROOT = env_string(
     "DELTACAT_ROOT",
-    os.path.join(os.getcwd(), ".deltacat"),
+    "",
 )
 
 # CLI Args

deltacat/env.py

@@ -1,3 +1,4 @@
+import argparse
 import os
 import logging
 from typing import Dict, Any
@@ -49,3 +50,12 @@ def create_ray_runtime_environment() -> Dict[str, Any]:
             "env_vars": worker_env_vars,
         }
     return runtime_environment
+
+
+def store_cli_args_in_os_environ(script_args_list=[]):
+    parser = argparse.ArgumentParser()
+    for args, kwargs in script_args_list:
+        parser.add_argument(*args, **kwargs)
+    args = parser.parse_args()
+    print(f"Command Line Arguments: {args}")
+    os.environ.update(vars(args))

deltacat/examples/basic_logging.py

@@ -6,9 +6,7 @@ import ray
 
 from deltacat import logs
 from deltacat.constants import DELTACAT_APP_LOG_DIR, DELTACAT_SYS_LOG_DIR
-from deltacat.examples.common.fixtures import (
-    store_cli_args_in_os_environ,
-)
+from env import store_cli_args_in_os_environ
 from deltacat.env import create_ray_runtime_environment
 
 # initialize the driver logger

deltacat/examples/{iceberg → experimental/iceberg}/iceberg_bucket_writer.py

@@ -9,10 +9,8 @@ import deltacat as dc
 
 from deltacat import logs
 from deltacat import IcebergCatalog
-from deltacat.catalog.iceberg import IcebergCatalogConfig
-from deltacat.examples.common.fixtures import (
-    store_cli_args_in_os_environ,
-)
+from deltacat.experimental.catalog.iceberg import IcebergCatalogConfig
+from env import store_cli_args_in_os_environ
 
 from pyiceberg.schema import (
     Schema,
@@ -23,7 +21,7 @@ from pyiceberg.schema import (
 from pyiceberg.partitioning import PartitionSpec, PartitionField
 from pyiceberg.transforms import BucketTransform
 
-from deltacat.storage.iceberg.model import (
+from deltacat.experimental.storage.iceberg.model import (
     SchemaMapper,
     PartitionSchemeMapper,
 )

deltacat/examples/{iceberg → experimental/iceberg}/iceberg_reader.py

@@ -4,9 +4,7 @@ import deltacat as dc
 
 from deltacat import logs
 from deltacat import IcebergCatalog
-from deltacat.examples.common.fixtures import (
-    store_cli_args_in_os_environ,
-)
+from env import store_cli_args_in_os_environ
 
 from pyiceberg.schema import (
     Schema,
@@ -22,7 +20,7 @@ from pyiceberg.transforms import DayTransform, IdentityTransform
 from pyiceberg.table.sorting import SortField, SortOrder
 
 from deltacat.exceptions import TableAlreadyExistsError
-from deltacat.storage.iceberg.model import (
+from deltacat.experimental.storage.iceberg.model import (
     SchemaMapper,
     PartitionSchemeMapper,
     SortSchemeMapper,

deltacat/examples/indexer/indexer.py

@@ -59,8 +59,8 @@ def run(
             "use_pyarrow": True,  # use the native pyarrow reader
         },
         # writer arguments to pass to the default writer (polars)
-        # for the given parquet-based datasink, it accepts the same
-        # arguments as polars.DataFrame.write_parquet except for `file`
+        # for the given parquet-based datasink, it generally accepts the same
+        # arguments as polars.DataFrame.write_{dest-type} except for `file`
         writer_args={
             "compression": "lz4",  # faster compression & decompression
             # "compression": "zstd",  # better compression ratio

deltacat/examples/indexer/job_runner.py

@@ -64,8 +64,7 @@ def run_sync(
     cluster_cfg_file_path = working_dir.joinpath(cloud).joinpath("deltacat.yaml")
     client = job_client(cluster_cfg_file_path, restart_ray=restart_ray)
     job_number = 0
-    while jobs_to_submit > 0:
-        jobs_to_submit -= 1
+    while job_number < jobs_to_submit:
         job_dest = dest + f".{job_number}"
         job_run_result = client.run_job(
             # Entrypoint shell command to execute

deltacat/experimental/catalog/iceberg/__init__.py

@@ -0,0 +1,6 @@
+from deltacat.experimental.catalog.iceberg.iceberg_catalog_config import (
+    IcebergCatalogConfig,
+)
+import deltacat.experimental.catalog.iceberg.impl as IcebergCatalog
+
+__all__ = ["IcebergCatalogConfig", "IcebergCatalog"]

deltacat/{catalog → experimental/catalog}/iceberg/iceberg_catalog_config.py

@@ -15,7 +15,7 @@ class IcebergCatalogConfig:
 
     This configuration is passed through to PyIceberg by invoking load_catalog.
     The Properties provided must match properties accepted by PyIceberg for each catalog type
-    See: :func:`deltacat.catalog.iceberg.initialize`
+    See: :func:`deltacat.experimental.catalog.iceberg.initialize`
 
     Attributes:
         type: The PyIceberg Catalog instance

deltacat/{catalog → experimental/catalog}/iceberg/impl.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 
 from typing import Any, Dict, List, Optional, Union
 
@@ -7,13 +8,19 @@ from daft.daft import ScanOperatorHandle, StorageConfig
 from daft.logical.builder import LogicalPlanBuilder
 
 from deltacat import logs
+from deltacat.catalog.model.catalog import Catalog
 from deltacat.catalog.model.table_definition import TableDefinition
-from deltacat.daft.daft_scan import DeltaCatScanOperator
+from deltacat.utils.daft import DeltaCatScanOperator
 from deltacat.exceptions import TableAlreadyExistsError
-from deltacat.storage.iceberg.iceberg_scan_planner import IcebergScanPlanner
-from deltacat.storage.iceberg.model import PartitionSchemeMapper, SchemaMapper
+from deltacat.experimental.storage.iceberg.iceberg_scan_planner import (
+    IcebergScanPlanner,
+)
+from deltacat.experimental.storage.iceberg.model import (
+    PartitionSchemeMapper,
+    SchemaMapper,
+)
 from deltacat.storage.model.partition import PartitionScheme
-from deltacat.storage.iceberg.impl import _get_native_catalog
+from deltacat.experimental.storage.iceberg.impl import _get_native_catalog
 from deltacat.storage.model.sort_key import SortScheme
 from deltacat.storage.model.list_result import ListResult
 from deltacat.storage.model.namespace import Namespace, NamespaceProperties
@@ -26,20 +33,31 @@ from deltacat.storage.model.types import (
     LocalTable,
     StreamFormat,
 )
-from deltacat.storage.iceberg import impl as IcebergStorage
+from deltacat.experimental.storage.iceberg import impl as IcebergStorage
 from deltacat.types.media import ContentType
 from deltacat.types.tables import TableWriteMode
 from deltacat.constants import DEFAULT_NAMESPACE
-from deltacat.catalog.iceberg.iceberg_catalog_config import IcebergCatalogConfig
+from deltacat.experimental.catalog.iceberg.iceberg_catalog_config import (
+    IcebergCatalogConfig,
+)
 
-from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.catalog import Catalog as PyIcebergCatalog, load_catalog
 from pyiceberg.transforms import BucketTransform
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
+IcebergCatalog = sys.modules[__name__]
+
+
+def from_config(config: IcebergCatalogConfig, *args, **kwargs) -> Catalog:
+    """
+    Factory method to construct a catalog from Iceberg catalog configuration.
+    """
+    return Catalog(config, impl=IcebergCatalog, *args, **kwargs)
+
 
 # catalog functions
-def initialize(*args, config: IcebergCatalogConfig, **kwargs) -> Catalog:
+def initialize(config: IcebergCatalogConfig, **kwargs) -> PyIcebergCatalog:
     """
     Initializes an Iceberg catalog with the given config.
 
@@ -123,7 +141,7 @@ def write_to_table(
                     )
                     # TODO(pdames): only append s3:// to output file paths when writing to S3!
                     out_file_paths = [f"s3://{val}" for val in out_df.to_arrow()[0]]
-                    from deltacat.catalog.iceberg import overrides
+                    from deltacat.experimental.catalog.iceberg import overrides
 
                     overrides.append(
                         table_definition.table.native_object,

deltacat/{storage → experimental/storage}/iceberg/iceberg_scan_planner.py

@@ -5,7 +5,7 @@ from deltacat.storage.model.scan.push_down import Pushdown
 from deltacat.storage.model.scan.scan_plan import ScanPlan
 from deltacat.storage.model.scan.scan_task import FileScanTask, DataFile
 from deltacat.storage.util.scan_planner import ScanPlanner
-from deltacat.storage.iceberg.impl import _try_load_iceberg_table
+from deltacat.experimental.storage.iceberg.impl import _try_load_iceberg_table
 
 
 class IcebergScanPlanner(ScanPlanner):

deltacat/{storage → experimental/storage}/iceberg/impl.py

@@ -32,7 +32,7 @@ from deltacat.storage import (
     NamespaceProperties,
 )
 from deltacat.storage.model.manifest import Manifest
-from deltacat.storage.iceberg.model import (
+from deltacat.experimental.storage.iceberg.model import (
     SchemaMapper,
     PartitionSchemeMapper,
     SortSchemeMapper,

deltacat/experimental/storage/rivulet/__init__.py

@@ -0,0 +1,11 @@
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.schema.schema import Field
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.schema.schema import Datatype
+
+__all__ = [
+    "Schema",
+    "Field",
+    "Dataset",
+    "Datatype",
+]

deltacat/{storage → experimental/storage}/rivulet/arrow/serializer.py

@@ -2,10 +2,13 @@ from abc import ABC, abstractmethod
 from typing import Iterator, List, Any
 import pyarrow as pa
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.serializer import DataSerializer, MEMTABLE_DATA
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.serializer import (
+    DataSerializer,
+    MEMTABLE_DATA,
+)
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 
 class ArrowSerializer(DataSerializer, ABC):

deltacat/{storage → experimental/storage}/rivulet/dataset.py

@@ -24,19 +24,23 @@ from deltacat.storage.model.shard import Shard, ShardingStrategy
 from deltacat.storage.model.stream import Stream, StreamLocator
 from deltacat.storage.model.transaction import TransactionOperationList
 from deltacat.storage.model.types import CommitState, StreamFormat
-from deltacat.storage.rivulet.fs.file_store import FileStore
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.reader.dataset_metastore import DatasetMetastore
-from deltacat.storage.rivulet import Schema, Field
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+from deltacat.experimental.storage.rivulet import Schema, Field
 from deltacat.utils.export import export_dataset
 from .schema.schema import Datatype
 
-from deltacat.storage.rivulet.reader.data_scan import DataScan
-from deltacat.storage.rivulet.reader.dataset_reader import DatasetReader
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
+from deltacat.experimental.storage.rivulet.reader.data_scan import DataScan
+from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
 
-from deltacat.storage.rivulet.writer.dataset_writer import DatasetWriter
-from deltacat.storage.rivulet.writer.memtable_dataset_writer import (
+from deltacat.experimental.storage.rivulet.writer.dataset_writer import DatasetWriter
+from deltacat.experimental.storage.rivulet.writer.memtable_dataset_writer import (
     MemtableDatasetWriter,
 )
 

deltacat/{storage → experimental/storage}/rivulet/dataset_executor.py

@@ -2,13 +2,16 @@ from __future__ import annotations
 
 from typing import List, Callable, Any
 
-from deltacat.storage.rivulet.field_group import FieldGroup
-from deltacat.storage.rivulet.mvp.Table import MvpTable
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.reader.data_scan import DataScan
-from deltacat.storage.rivulet.reader.dataset_metastore import DatasetMetastore
-from deltacat.storage.rivulet.reader.dataset_reader import DatasetReader
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
+from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.data_scan import DataScan
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
 
 
 class DatasetExecutor:
@@ -22,12 +25,10 @@ class DatasetExecutor:
 
     def __init__(
         self,
-        field_groups: List[FieldGroup],
         schema: Schema,
         metastore: DatasetMetastore,
     ):
         self.effective_schema: Schema = schema.__deepcopy__()
-        self.field_groups = field_groups
         self.output: MvpTable | None = None
         self._metastore = metastore
 
@@ -64,18 +65,9 @@ class DatasetExecutor:
 
         TODO for now this is doing dumb in-memory implementation and later this is going to be replaced by rust library
         """
-        if len(self.field_groups) == 1:
-            return self._read_as_mvp_table(schema, self.field_groups[0])
-        else:
-            ds1 = self._read_as_mvp_table(schema, self.field_groups[0])
-            ds2 = self._read_as_mvp_table(schema, self.field_groups[1])
-            merged = MvpTable.merge(ds1, ds2, schema.primary_key.name)
-            for i in range(2, len(self.field_groups)):
-                ds_i = self._read_as_mvp_table(schema, self.field_groups[i])
-                merged = MvpTable.merge(merged, ds_i, schema.primary_key.name)
-            return merged
+        return self._read_as_mvp_table(schema)
 
-    def _read_as_mvp_table(self, schema: Schema, field_group: FieldGroup):
+    def _read_as_mvp_table(self, schema: Schema):
         data = list(
             DataScan(
                 schema, QueryExpression(), DatasetReader(self._metastore)

deltacat/experimental/storage/rivulet/feather/__init__.py

@@ -0,0 +1,7 @@
+# TODO later on this will be moved to a dedicated package
+from deltacat.experimental.storage.rivulet.feather.file_reader import FeatherFileReader
+from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
+    FileReaderRegistrar,
+)
+
+FileReaderRegistrar.register_reader("feather", FeatherFileReader)

deltacat/{storage → experimental/storage}/rivulet/feather/file_reader.py

@@ -5,15 +5,17 @@ from typing import Optional
 import pyarrow.ipc
 from pyarrow import RecordBatch, RecordBatchFileReader
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.reader.data_reader import (
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
     RowAndKey,
     FileReader,
     FILE_FORMAT,
 )
-from deltacat.storage.rivulet.reader.pyarrow_data_reader import RecordBatchRowIndex
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
+    RecordBatchRowIndex,
+)
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 
 
 class FeatherFileReader(FileReader[RecordBatchRowIndex]):

deltacat/{storage → experimental/storage}/rivulet/feather/serializer.py

@@ -3,10 +3,10 @@ from typing import List
 import pyarrow as pa
 from pyarrow import feather
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.arrow.serializer import ArrowSerializer
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.arrow.serializer import ArrowSerializer
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 
 class FeatherDataSerializer(ArrowSerializer):

deltacat/{storage → experimental/storage}/rivulet/fs/file_provider.py

@@ -3,9 +3,9 @@ import time
 from typing import List, Generator
 
 from deltacat.storage.model.partition import PartitionLocator
-from deltacat.storage.rivulet.fs.file_store import FileStore
-from deltacat.storage.rivulet.fs.input_file import InputFile
-from deltacat.storage.rivulet.fs.output_file import OutputFile
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
 from deltacat.utils.metafile_locator import _find_partition_path
 
 

deltacat/{storage → experimental/storage}/rivulet/fs/file_store.py

@@ -4,8 +4,8 @@ from pyarrow.fs import FileSystem, FileType, FileSelector
 # TODO(deltacat): Rely on deltacat implementation to resolve path and filesystem.
 from ray.data.datasource.path_util import _resolve_paths_and_filesystem
 
-from deltacat.storage.rivulet.fs.input_file import FSInputFile
-from deltacat.storage.rivulet.fs.output_file import FSOutputFile
+from deltacat.experimental.storage.rivulet.fs.input_file import FSInputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import FSOutputFile
 
 
 class FileStore:

deltacat/{storage → experimental/storage}/rivulet/fs/output_file.py

@@ -5,7 +5,7 @@ from typing import Protocol
 
 from pyarrow.fs import FileSystem, FileType
 
-from deltacat.storage.rivulet.fs.input_file import FSInputFile, InputFile
+from deltacat.experimental.storage.rivulet.fs.input_file import FSInputFile, InputFile
 
 
 class OutputStream(Protocol):  # pragma: no cover

deltacat/{storage → experimental/storage}/rivulet/logical_plan.py

@@ -1,9 +1,9 @@
 from dataclasses import dataclass
 from typing import List, Callable, Any, Protocol
 
-from deltacat.storage.rivulet.dataset_executor import DatasetExecutor
-from deltacat.storage.rivulet.mvp.Table import MvpTable
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.dataset_executor import DatasetExecutor
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
 
 
 class DatasetOperation(Protocol):
@@ -99,7 +99,7 @@ class LogicalPlan:
         self.operations.append(CollectOperation())
         return self
 
-    def execute(self, executor: DatasetExecutor) -> "MvpTable":
+    def execute(self, executor: DatasetExecutor) -> MvpTable:
         for operation in self.operations:
             operation.visit(executor)
         return executor.output

deltacat/{storage → experimental/storage}/rivulet/metastore/delta.py

@@ -19,7 +19,7 @@ from deltacat.storage.model.partition import PartitionLocator
 from deltacat.storage.model.transaction import TransactionOperationList
 
 from deltacat.storage.model.types import StreamFormat
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet import Schema
 
 StreamPosition = int
 """The stream position for creating a consistent ordering of manifests."""

deltacat/{storage → experimental/storage}/rivulet/metastore/json_sst.py

@@ -4,9 +4,9 @@ import json
 from itertools import zip_longest
 from typing import List
 
-from deltacat.storage.rivulet.fs.input_file import InputFile
-from deltacat.storage.rivulet.fs.output_file import OutputFile
-from deltacat.storage.rivulet.metastore.sst import (
+from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
+from deltacat.experimental.storage.rivulet.metastore.sst import (
     SSTWriter,
     SSTableRow,
     SSTReader,

deltacat/{storage → experimental/storage}/rivulet/metastore/sst.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from typing import Protocol, Any, List
 
-from deltacat.storage.rivulet.fs.input_file import InputFile
-from deltacat.storage.rivulet.fs.output_file import OutputFile
+from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
 
 
 @dataclass(frozen=True)

deltacat/{storage → experimental/storage}/rivulet/metastore/sst_interval_tree.py

@@ -8,9 +8,9 @@ from typing import Any, Dict, Set, List, FrozenSet, Iterable, TypeVar, NamedTupl
 
 from intervaltree import Interval, IntervalTree
 
-from deltacat.storage.rivulet.metastore.delta import DeltaContext
-from deltacat.storage.rivulet.metastore.sst import SSTable, SSTableRow
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTable, SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
 
 T = TypeVar("T")
 

deltacat/experimental/storage/rivulet/parquet/__init__.py

@@ -0,0 +1,7 @@
+# TODO later on this will be moved to a dedicated package
+from deltacat.experimental.storage.rivulet.parquet.file_reader import ParquetFileReader
+from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
+    FileReaderRegistrar,
+)
+
+FileReaderRegistrar.register_reader("parquet", ParquetFileReader)

deltacat/{storage → experimental/storage}/rivulet/parquet/file_reader.py

@@ -4,15 +4,17 @@ from typing import Optional
 
 from pyarrow import RecordBatch
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.reader.data_reader import (
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
     RowAndKey,
     FileReader,
     FILE_FORMAT,
 )
-from deltacat.storage.rivulet.reader.pyarrow_data_reader import RecordBatchRowIndex
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
+    RecordBatchRowIndex,
+)
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 import pyarrow.parquet as pq
 import pyarrow as pa
 

deltacat/{storage → experimental/storage}/rivulet/parquet/serializer.py

@@ -3,11 +3,11 @@ from typing import List, Any
 import pyarrow as pa
 from pyarrow.parquet import FileMetaData
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.arrow.serializer import ArrowSerializer
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.arrow.serializer import ArrowSerializer
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 
 class ParquetDataSerializer(ArrowSerializer):

deltacat/{storage → experimental/storage}/rivulet/reader/block_scanner.py

@@ -15,19 +15,30 @@ from typing import (
     AbstractSet,
 )
 
-from deltacat.storage.rivulet.metastore.delta import DeltaContext
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.metastore.sst_interval_tree import (
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
     OrderedBlockGroups,
     BlockGroup,
     Block,
 )
-from deltacat.storage.rivulet.reader.data_reader import RowAndKey, FileReader
-from deltacat.storage.rivulet.reader.dataset_metastore import DatasetMetastore
-from deltacat.storage.rivulet.reader.pyarrow_data_reader import ArrowDataReader
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet.reader.reader_type_registrar import FileReaderRegistrar
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
+    RowAndKey,
+    FileReader,
+)
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
+    ArrowDataReader,
+)
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
+    FileReaderRegistrar,
+)
+from deltacat.experimental.storage.rivulet import Schema
 from deltacat import logs
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))