deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/types/tables.py

@@ -1,11 +1,36 @@
+# Allow classes to use self-referencing Type hints in Python 3.7.
+from __future__ import annotations
+
+import logging
+import multiprocessing
 from enum import Enum
-from typing import Callable, Dict, Type, Union
+from functools import partial
+from typing import (
+    Callable,
+    Dict,
+    Type,
+    Union,
+    Optional,
+    Any,
+    List,
+    Tuple,
+    TYPE_CHECKING,
+)
+from uuid import uuid4
 
+import daft
 import numpy as np
 import pandas as pd
+import polars as pl
 import pyarrow as pa
+import pyarrow.dataset as ds
+import pyarrow.fs
 import pyarrow.parquet as papq
-from ray.data.dataset import Dataset
+import ray
+from ray.data.block import Block, BlockMetadata, BlockAccessor
+from ray.data._internal.pandas_block import PandasBlockSchema
+from ray.data.dataset import Dataset as RayDataset, MaterializedDataset
+from ray.data.datasource import FilenameProvider
 from ray.data.read_api import (
     from_arrow,
     from_arrow_refs,
@@ -13,74 +38,506 @@ from ray.data.read_api import (
     from_pandas,
     from_pandas_refs,
 )
+from tenacity import (
+    Retrying,
+    wait_random_exponential,
+    stop_after_delay,
+    retry_if_exception_type,
+)
 
-import deltacat.storage as dcs
-from deltacat.types.media import TableType, DistributedDatasetType
+from deltacat.compute.compactor_v2.constants import MAX_RECORDS_PER_COMPACTED_FILE
+from deltacat import logs
+from deltacat.constants import (
+    UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY,
+    RETRYABLE_TRANSIENT_ERRORS,
+    DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY,
+)
+from deltacat.storage.model.types import (
+    Dataset,
+    LocalTable,
+    DistributedDataset,
+    LocalDataset,
+)
+from deltacat.storage.model.schema import SchemaConsistencyType
+from deltacat.types.media import (
+    DatasetType,
+    DistributedDatasetType,
+    ContentType,
+    EXPLICIT_COMPRESSION_CONTENT_TYPES,
+    ContentEncoding,
+    CONTENT_TYPE_TO_EXT,
+    CONTENT_ENCODING_TO_EXT,
+)
 from deltacat.utils import numpy as np_utils
 from deltacat.utils import pandas as pd_utils
+from deltacat.utils import polars as pl_utils
 from deltacat.utils import pyarrow as pa_utils
-from deltacat.utils import daft as daft_utils
 from deltacat.utils.ray_utils import dataset as ds_utils
+from deltacat.storage.model.manifest import (
+    ManifestEntryList,
+    ManifestEntry,
+    ManifestMeta,
+    EntryParams,
+    EntryType,
+    Manifest,
+)
+from deltacat.exceptions import (
+    RetryableError,
+    RetryableUploadTableError,
+    NonRetryableUploadTableError,
+    categorize_errors,
+    RetryableDownloadTableError,
+    NonRetryableDownloadTableError,
+)
+from deltacat.utils.common import ReadKwargsProvider
+from deltacat.types.partial_download import PartialFileDownloadParams
+from deltacat.utils.ray_utils.concurrency import invoke_parallel
 
-TABLE_TYPE_TO_READER_FUNC: Dict[int, Callable] = {
-    TableType.PYARROW_PARQUET.value: pa_utils.s3_file_to_parquet,
-    TableType.PYARROW.value: pa_utils.s3_file_to_table,
-    TableType.PANDAS.value: pd_utils.s3_file_to_dataframe,
-    TableType.NUMPY.value: np_utils.s3_file_to_ndarray,
+if TYPE_CHECKING:
+    from deltacat.storage.model.schema import Schema
+    from deltacat.storage.model.table import Table
+    from deltacat.storage.model.table_version import TableVersion
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+TABLE_TYPE_TO_READER_FUNC: Dict[str, Callable] = {
+    DatasetType.PYARROW_PARQUET.value: pa_utils.file_to_parquet,
+    DatasetType.PYARROW.value: pa_utils.file_to_table,
+    DatasetType.PANDAS.value: pd_utils.file_to_dataframe,
+    DatasetType.NUMPY.value: np_utils.file_to_ndarray,
+    DatasetType.POLARS.value: pl_utils.file_to_dataframe,
 }
 
+
 TABLE_CLASS_TO_WRITER_FUNC: Dict[
-    Type[Union[dcs.LocalTable, dcs.DistributedDataset]], Callable
+    Type[Union[LocalTable, DistributedDataset]], Callable
 ] = {
     pa.Table: pa_utils.table_to_file,
     pd.DataFrame: pd_utils.dataframe_to_file,
+    pl.DataFrame: pl_utils.dataframe_to_file,
     np.ndarray: np_utils.ndarray_to_file,
-    Dataset: ds_utils.dataset_to_file,
+    RayDataset: ds_utils.dataset_to_file,
+    MaterializedDataset: ds_utils.dataset_to_file,
 }
 
 TABLE_CLASS_TO_SLICER_FUNC: Dict[
-    Type[Union[dcs.LocalTable, dcs.DistributedDataset]], Callable
+    Type[Union[LocalTable, DistributedDataset]], Callable
 ] = {
     pa.Table: pa_utils.slice_table,
     pd.DataFrame: pd_utils.slice_dataframe,
+    pl.DataFrame: pl_utils.slice_table,
     np.ndarray: np_utils.slice_ndarray,
-    Dataset: ds_utils.slice_dataset,
+    RayDataset: ds_utils.slice_dataset,
+    MaterializedDataset: ds_utils.slice_dataset,
 }
 
 TABLE_CLASS_TO_SIZE_FUNC: Dict[
-    Type[Union[dcs.LocalTable, dcs.DistributedDataset]], Callable
+    Type[Union[LocalTable, DistributedDataset]], Callable
 ] = {
     pa.Table: pa_utils.table_size,
     papq.ParquetFile: pa_utils.parquet_file_size,
     pd.DataFrame: pd_utils.dataframe_size,
+    pl.DataFrame: pl_utils.dataframe_size,
     np.ndarray: np_utils.ndarray_size,
-    Dataset: ds_utils.dataset_size,
+    RayDataset: ds_utils.dataset_size,
+    MaterializedDataset: ds_utils.dataset_size,
+}
+
+TABLE_CLASS_TO_COLUMN_NAMES_FUNC: Dict[
+    Type[Union[LocalTable, DistributedDataset]], Callable
+] = {
+    pa.Table: lambda table: table.schema.names,
+    papq.ParquetFile: lambda table: table.schema.names,
+    pd.DataFrame: lambda table: table.columns.tolist(),
+    pl.DataFrame: lambda table: table.columns,
+    np.ndarray: lambda table: [f"{i}" for i in range(table.shape[1])],
+    daft.DataFrame: lambda table: table.column_names,
+    RayDataset: lambda table: table.schema().names,
+    MaterializedDataset: lambda table: table.schema().names,
+}
+
+TABLE_CLASS_TO_SCHEMA_FUNC: Dict[
+    Type[Union[LocalTable, DistributedDataset]], Callable
+] = {
+    pa.Table: lambda table: table.schema,
+    papq.ParquetFile: lambda table: table.schema_arrow,
+    pd.DataFrame: lambda table: pa.Schema.from_pandas(table),
+    pl.DataFrame: lambda table: table.to_arrow().schema,
+    np.ndarray: lambda table: pa.Schema.from_pandas(pd.DataFrame(table)),
+    daft.DataFrame: lambda table: table.schema().to_pyarrow_schema(),
+    RayDataset: lambda table: table.schema().base_schema,
+    MaterializedDataset: lambda table: table.schema().base_schema,
+}
+
+TABLE_TYPE_TO_EMPTY_TABLE_FUNC: Dict[str, Callable] = {
+    DatasetType.PYARROW.value: lambda: pa.Table.from_pydict({}),
+    DatasetType.PANDAS.value: lambda: pd.DataFrame(),
+    DatasetType.POLARS.value: lambda: pl.DataFrame(),
+    DatasetType.NUMPY.value: lambda: np.array([]),
+    DatasetType.DAFT.value: lambda: daft.DataFrame(),
+    DatasetType.RAY_DATASET.value: lambda: ray.data.from_items([]),
+    MaterializedDataset: lambda: ray.data.from_items([]),
 }
 
-TABLE_CLASS_TO_TABLE_TYPE: Dict[Type[dcs.LocalTable], str] = {
-    pa.Table: TableType.PYARROW.value,
-    papq.ParquetFile: TableType.PYARROW_PARQUET.value,
-    pd.DataFrame: TableType.PANDAS.value,
-    np.ndarray: TableType.NUMPY.value,
+
+def _numpy_array_to_pyarrow(table: np.ndarray, schema: pa.Schema) -> pa.Table:
+    """Convert NumPy array to PyArrow Table via Pandas for complex type support."""
+    # Convert NumPy -> Pandas -> PyArrow to handle complex types like structs
+    # This follows the same path as Pandas conversion which handles all complex types properly
+    pandas_df = _numpy_array_to_pandas(table, schema=schema)
+    return pa.Table.from_pandas(pandas_df, schema=schema)
+
+
+def _numpy_array_to_pandas(
+    table: np.ndarray, *, schema: Optional[pa.Schema] = None, **kwargs
+) -> pd.DataFrame:
+    """Convert NumPy array to pandas DataFrame."""
+    if schema and isinstance(schema, pa.Schema):
+        if table.ndim == 1:
+            # 1D array: single column
+            column_names = [schema.names[0]] if schema.names else ["0"]
+            return pd.DataFrame({column_names[0]: table}, **kwargs)
+        elif table.ndim == 2:
+            # 2D array: multiple columns
+            column_names = (
+                schema.names
+                if len(schema.names) == table.shape[1]
+                else [f"{i}" for i in range(table.shape[1])]
+            )
+            return pd.DataFrame(table, columns=column_names, **kwargs)
+        else:
+            raise ValueError(
+                f"NumPy arrays with {table.ndim} dimensions are not supported. "
+                f"Only 1D and 2D arrays are supported."
+            )
+
+    # Fallback to generic column names
+    return pd.DataFrame(table, **kwargs)
+
+
+def _ray_dataset_to_pyarrow(table, *, schema, **kwargs):
+    """Convert Ray Dataset to PyArrow tables and concatenate."""
+    arrow_refs = table.to_arrow_refs(**kwargs)
+    arrow_tables = ray.get(arrow_refs)
+    if len(arrow_tables) == 1:
+        return arrow_tables[0]
+    # Unify schemas to support schema evolution across blocks/files
+    try:
+        return pa.concat_tables(
+            arrow_tables, promote_options="permissive", unify_schemas=True
+        )
+    except TypeError:
+        return pa.concat_tables(arrow_tables, promote_options="permissive")
+
+
+TABLE_CLASS_TO_PYARROW_FUNC: Dict[
+    Type[Union[LocalTable, DistributedDataset]], Callable
+] = {
+    pa.Table: lambda table, *, schema, **kwargs: table,
+    papq.ParquetFile: lambda table, *, schema, **kwargs: table.read(**kwargs),
+    pd.DataFrame: lambda table, *, schema, **kwargs: pa.Table.from_pandas(
+        table, schema=schema, preserve_index=False, **kwargs
+    ),
+    pl.DataFrame: lambda table, *, schema, **kwargs: pl.DataFrame.to_arrow(
+        table, **kwargs
+    ),
+    np.ndarray: lambda table, *, schema, **kwargs: _numpy_array_to_pyarrow(
+        table, schema, **kwargs
+    ),
+    RayDataset: _ray_dataset_to_pyarrow,
+    MaterializedDataset: _ray_dataset_to_pyarrow,
+    daft.DataFrame: lambda table, *, schema, **kwargs: table.to_arrow(**kwargs),
+}
+
+TABLE_CLASS_TO_PANDAS_FUNC: Dict[
+    Type[Union[LocalTable, DistributedDataset]], Callable
+] = {
+    pa.Table: lambda table, *, schema=None, **kwargs: table.to_pandas(**kwargs),
+    papq.ParquetFile: lambda table, *, schema=None, **kwargs: table.read(
+        **kwargs
+    ).to_pandas(**kwargs),
+    pd.DataFrame: lambda table, *, schema=None, **kwargs: table,
+    pl.DataFrame: lambda table, *, schema=None, **kwargs: table.to_pandas(**kwargs),
+    np.ndarray: lambda table, *, schema=None, **kwargs: _numpy_array_to_pandas(
+        table, schema=schema, **kwargs
+    ),
+    RayDataset: lambda table, *, schema=None, **kwargs: table.to_pandas(**kwargs),
+    MaterializedDataset: lambda table, *, schema=None, **kwargs: table.to_pandas(
+        **kwargs
+    ),
+    daft.DataFrame: lambda table, *, schema=None, **kwargs: table.to_pandas(**kwargs),
+}
+
+
+def _pyarrow_to_polars(pa_table: pa.Table, **kwargs) -> pl.DataFrame:
+    """Convert PyArrow table to Polars DataFrame with clean schema."""
+    # PyArrow metadata can contain invalid UTF-8 sequences that cause Polars to raise an error
+    # Create a new table without metadata that might contain invalid UTF-8
+    clean_schema = pa.schema(
+        [
+            pa.field(field.name, field.type, nullable=field.nullable)
+            for field in pa_table.schema
+        ]
+    )
+    clean_table = pa.Table.from_arrays(pa_table.columns, schema=clean_schema)
+    return pl.from_arrow(clean_table, **kwargs)
+
+
+def _pyarrow_to_numpy(pa_table: pa.Table, **kwargs) -> np.ndarray:
+    """Convert PyArrow table to numpy array."""
+    if pa_table.num_columns == 1:
+        return pa_table.column(0).to_numpy(**kwargs)
+    else:
+        return pa_table.to_pandas().values
+
+
+def _pandas_to_numpy(pd_df: pd.DataFrame, **kwargs) -> np.ndarray:
+    """Convert Pandas DataFrame to numpy array."""
+    if len(pd_df.columns) == 1:
+        return pd_df.iloc[:, 0].to_numpy(**kwargs)
+    else:
+        return pd_df.values
+
+
+DATASET_TYPE_FROM_PYARROW: Dict[DatasetType, Callable[[pa.Table, Dataset], Any]] = {
+    DatasetType.PYARROW: lambda pa_table, **kwargs: pa_table,
+    DatasetType.PANDAS: lambda pa_table, **kwargs: pa_table.to_pandas(**kwargs),
+    DatasetType.POLARS: lambda pa_table, **kwargs: _pyarrow_to_polars(
+        pa_table, **kwargs
+    ),
+    DatasetType.DAFT: lambda pa_table, **kwargs: daft.from_arrow(pa_table, **kwargs),
+    DatasetType.NUMPY: lambda pa_table, **kwargs: _pyarrow_to_numpy(pa_table, **kwargs),
+    DatasetType.RAY_DATASET: lambda pa_table, **kwargs: ray.data.from_arrow(pa_table),
+    DatasetType.PYARROW_PARQUET: lambda pa_table, **kwargs: pa_table,  # ParquetFile is read as PyArrow Table
+}
+
+
+DATASET_TYPE_FROM_PANDAS: Dict[DatasetType, Callable[[pd.DataFrame, Dataset], Any]] = {
+    DatasetType.PANDAS: lambda pd_df, **kwargs: pd_df,
+    DatasetType.PYARROW: lambda pd_df, **kwargs: pa.Table.from_pandas(pd_df, **kwargs),
+    DatasetType.POLARS: lambda pd_df, **kwargs: pl.from_pandas(pd_df, **kwargs),
+    DatasetType.DAFT: lambda pd_df, **kwargs: daft.from_pandas(pd_df, **kwargs),
+    DatasetType.NUMPY: lambda pd_df, **kwargs: _pandas_to_numpy(pd_df, **kwargs),
+    DatasetType.RAY_DATASET: lambda pd_df, **kwargs: ray.data.from_pandas(
+        pd_df, **kwargs
+    ),
+}
+
+
+def append_column_to_parquet_file(
+    parquet_file: papq.ParquetFile,
+    column_name: str,
+    column_value: Any,
+) -> pa.Table:
+    """
+    Append a column to a ParquetFile by converting to PyArrow Table first.
+
+    Args:
+        parquet_file: The ParquetFile to add column to
+        column_name: Name of the new column
+        column_value: Value to populate in all rows of the new column
+
+    Returns:
+        PyArrow Table with the new column
+    """
+    # Convert ParquetFile to Table
+    table = parquet_file.read()
+
+    # Use the existing PyArrow append column function
+    num_rows = table.num_rows
+    column_array = pa.array([column_value] * num_rows)
+    return table.append_column(column_name, column_array)
+
+
+TABLE_CLASS_TO_APPEND_COLUMN_FUNC: Dict[
+    Type[Union[LocalTable, DistributedDataset]], Callable
+] = {
+    pa.Table: pa_utils.append_column_to_table,
+    papq.ParquetFile: append_column_to_parquet_file,
+    pd.DataFrame: pd_utils.append_column_to_dataframe,
+    pl.DataFrame: pl_utils.append_column_to_table,
+    np.ndarray: np_utils.append_column_to_ndarray,
+}
+
+TABLE_CLASS_TO_SELECT_COLUMNS_FUNC: Dict[
+    Type[Union[LocalTable, DistributedDataset]], Callable
+] = {
+    pa.Table: pa_utils.select_columns,
+    pd.DataFrame: pd_utils.select_columns,
+    pl.DataFrame: pl_utils.select_columns,
+}
+
+TABLE_CLASS_TO_TABLE_TYPE: Dict[Union[LocalTable, DistributedDataset], str] = {
+    pa.Table: DatasetType.PYARROW.value,
+    papq.ParquetFile: DatasetType.PYARROW_PARQUET.value,
+    pl.DataFrame: DatasetType.POLARS.value,
+    pd.DataFrame: DatasetType.PANDAS.value,
+    np.ndarray: DatasetType.NUMPY.value,
+    daft.DataFrame: DatasetType.DAFT.value,
+    RayDataset: DatasetType.RAY_DATASET.value,
+    MaterializedDataset: DatasetType.RAY_DATASET.value,
 }
 
 TABLE_TYPE_TO_DATASET_CREATE_FUNC: Dict[str, Callable] = {
-    TableType.PYARROW.value: from_arrow,
-    TableType.PYARROW_PARQUET.value: from_arrow,
-    TableType.NUMPY.value: from_numpy,
-    TableType.PANDAS.value: from_pandas,
+    DatasetType.PYARROW.value: from_arrow,
+    DatasetType.PYARROW_PARQUET.value: from_arrow,
+    DatasetType.NUMPY.value: from_numpy,
+    DatasetType.PANDAS.value: from_pandas,
 }
 
 TABLE_TYPE_TO_DATASET_CREATE_FUNC_REFS: Dict[str, Callable] = {
-    TableType.PYARROW.value: from_arrow_refs,
-    TableType.PYARROW_PARQUET.value: from_arrow_refs,
-    TableType.NUMPY.value: from_numpy,
-    TableType.PANDAS.value: from_pandas_refs,
+    DatasetType.PYARROW.value: from_arrow_refs,
+    DatasetType.PYARROW_PARQUET.value: from_arrow_refs,
+    DatasetType.NUMPY.value: from_numpy,
+    DatasetType.PANDAS.value: from_pandas_refs,
+    DatasetType.POLARS.value: from_arrow_refs,  # We cast Polars to Arrow for Ray Datasets
+    DatasetType.RAY_DATASET.value: from_arrow_refs,  # Ray Datasets are created from Arrow refs
+}
+
+TABLE_TYPE_TO_CONCAT_FUNC: Dict[str, Callable] = {
+    DatasetType.PYARROW_PARQUET.value: pa_utils.concat_tables,
+    DatasetType.PYARROW.value: pa_utils.concat_tables,
+    DatasetType.PANDAS.value: pd_utils.concat_dataframes,
+    DatasetType.NUMPY.value: np_utils.concat_ndarrays,
+    DatasetType.POLARS.value: pl_utils.concat_dataframes,
 }
 
 
+def _infer_schema_from_numpy_array(data: np.ndarray) -> Schema:
+    """Infer schema from NumPy array."""
+    if data.ndim > 2:
+        raise ValueError(
+            f"NumPy arrays with {data.ndim} dimensions are not supported. "
+            f"Only 1D and 2D arrays are supported."
+        )
+    # Handle object dtype by converting to pandas first
+    df = pd.DataFrame(data)
+    arrow_schema = pa.Schema.from_pandas(df)
+
+    from deltacat.storage.model.schema import Schema
+
+    return Schema.of(schema=arrow_schema)
+
+
+def _infer_schema_from_ray_dataset(data: RayDataset) -> Schema:
+    """Infer schema from Ray Dataset."""
+    ray_schema = data.schema()
+    base_schema = ray_schema.base_schema
+
+    if isinstance(base_schema, pa.Schema):
+        arrow_schema = base_schema
+    elif isinstance(base_schema, PandasBlockSchema):
+        try:
+            dtype_dict = {
+                name: dtype for name, dtype in zip(base_schema.names, base_schema.types)
+            }
+            empty_df = pd.DataFrame(columns=base_schema.names).astype(dtype_dict)
+            arrow_schema = pa.Schema.from_pandas(empty_df)
+        except Exception as e:
+            raise ValueError(
+                f"Failed to convert Ray Dataset PandasBlockSchema to PyArrow schema: {e}"
+            )
+    else:
+        raise ValueError(
+            f"Unsupported Ray Dataset schema type: {type(base_schema)}. "
+            f"Expected PyArrow Schema or PandasBlockSchema, got {base_schema}"
+        )
+
+    from deltacat.storage.model.schema import Schema
+
+    return Schema.of(schema=arrow_schema)
+
+
+def _infer_schema_from_pandas_dataframe(data: pd.DataFrame) -> Schema:
+    """Infer schema from Pandas DataFrame."""
+    from deltacat.storage.model.schema import Schema
+
+    arrow_schema = pa.Schema.from_pandas(data)
+    return Schema.of(schema=arrow_schema)
+
+
+def _infer_schema_from_polars_dataframe(data: pl.DataFrame) -> Schema:
+    """Infer schema from Polars DataFrame."""
+    from deltacat.storage.model.schema import Schema
+
+    arrow_table = data.to_arrow()
+    return Schema.of(schema=arrow_table.schema)
+
+
+def _infer_schema_from_pyarrow(
+    data: Union[pa.Table, pa.RecordBatch, ds.Dataset]
+) -> Schema:
+    """Infer schema from PyArrow Table, RecordBatch, or Dataset."""
+    from deltacat.storage.model.schema import Schema
+
+    return Schema.of(schema=data.schema)
+
+
+def _infer_schema_from_daft_dataframe(data: daft.DataFrame) -> Schema:
+    """Infer schema from Daft DataFrame."""
+    from deltacat.storage.model.schema import Schema
+
+    daft_schema = data.schema()
+    arrow_schema = daft_schema.to_pyarrow_schema()
+    return Schema.of(schema=arrow_schema)
+
+
+TABLE_CLASS_TO_SCHEMA_INFERENCE_FUNC: Dict[
+    Type[Union[LocalTable, DistributedDataset], Callable]
+] = {
+    pd.DataFrame: _infer_schema_from_pandas_dataframe,
+    pl.DataFrame: _infer_schema_from_polars_dataframe,
+    pa.Table: _infer_schema_from_pyarrow,
+    pa.RecordBatch: _infer_schema_from_pyarrow,
+    ds.Dataset: _infer_schema_from_pyarrow,
+    RayDataset: _infer_schema_from_ray_dataset,
+    MaterializedDataset: _infer_schema_from_ray_dataset,  # MaterializedDataset uses same schema inference as RayDataset
+    daft.DataFrame: _infer_schema_from_daft_dataframe,
+    np.ndarray: _infer_schema_from_numpy_array,
+}
+
+
+def infer_table_schema(data: Union[LocalTable, DistributedDataset]) -> Schema:
+    """Infer schema from a table or dataset."""
+    infer_schema_func = _get_table_function(
+        data,
+        TABLE_CLASS_TO_SCHEMA_INFERENCE_FUNC,
+        "schema inference",
+    )
+    return infer_schema_func(data)
+
+
+def concat_tables(tables: List[LocalTable], table_type: DatasetType) -> LocalTable:
+    """
+    Concatenate a list of tables into a single table using the appropriate
+    concatenation function for the given table type.
+
+    Args:
+        tables: List of tables to concatenate
+        table_type: The DatasetType indicating which concatenation function to use
+
+    Returns:
+        Single concatenated table of the appropriate type
+
+    Raises:
+        ValueError: If no concatenation function is found for the table type
+    """
+    concat_func = _get_table_type_function(
+        table_type, TABLE_TYPE_TO_CONCAT_FUNC, "concatenation"
+    )
+    return concat_func(tables)
+
+
+def _daft_reader_wrapper(*args, **kwargs):
+    """Wrapper for daft reader with lazy import to avoid circular import."""
+    from deltacat.utils.daft import files_to_dataframe
+
+    return files_to_dataframe(*args, **kwargs)
+
+
 DISTRIBUTED_DATASET_TYPE_TO_READER_FUNC: Dict[int, Callable] = {
-    DistributedDatasetType.DAFT.value: daft_utils.s3_files_to_dataframe
+    DistributedDatasetType.DAFT.value: _daft_reader_wrapper,
 }
 
 
@@ -89,13 +546,14 @@ class TableWriteMode(str, Enum):
     Enum controlling how a given dataset will be written to a table.
 
     AUTO: CREATE if the table doesn't exist, APPEND if the table exists
-    without primary keys, and MERGE if the table exists with primary keys.
+    without merge keys, and MERGE if the table exists with merge keys.
     CREATE: Create the table if it doesn't exist, throw an error if it does.
     APPEND: Append to the table if it exists, throw an error if it doesn't.
     REPLACE: Replace existing table contents with the data to write.
-    MERGE: Insert, update, or delete records matching a given predicate.
-    Updates or inserts records based on the table's primary and sort keys by
+    MERGE: Insert or update records matching table merge keys.
+    Updates or inserts records based on the table's merge and sort keys by
     default.
+    DELETE: Delete records matching table merge keys.
     """
 
     AUTO = "auto"
@@ -103,29 +561,1869 @@ class TableWriteMode(str, Enum):
     APPEND = "append"
     REPLACE = "replace"
     MERGE = "merge"
+    DELETE = "delete"
 
 
-def get_table_length(table: Union[dcs.LocalTable, dcs.DistributedDataset]) -> int:
-    return len(table) if not isinstance(table, Dataset) else table.count()
+class SchemaEvolutionMode(str, Enum):
+    """
+    Enum controlling how schema changes are handled when writing to a table.
+
+    MANUAL: Schema changes must be explicitly handled by the user. New fields
+    not in the existing schema will cause an error.
+    AUTO: Schema changes are automatically handled. New fields are added to
+    the schema using the table's default_schema_consistency_type.
+    DISABLED: Schema changes are disabled. The schema that the table was
+    created with is immutable.
+    """
+
+    MANUAL = "manual"
+    AUTO = "auto"
+    DISABLED = "disabled"
+
+
+class TableProperty(str, Enum):
+    """
+    Enum defining known table property key names.
+    """
+
+    READ_OPTIMIZATION_LEVEL = "read_optimization_level"
+    RECORDS_PER_COMPACTED_FILE = "records_per_compacted_file"
+    APPENDED_RECORD_COUNT_COMPACTION_TRIGGER = (
+        "appended_record_count_compaction_trigger"
+    )
+    APPENDED_FILE_COUNT_COMPACTION_TRIGGER = "appended_file_count_compaction_trigger"
+    APPENDED_DELTA_COUNT_COMPACTION_TRIGGER = "appended_delta_count_compaction_trigger"
+    DEFAULT_COMPACTION_HASH_BUCKET_COUNT = "default_compaction_hash_bucket_count"
+    SCHEMA_EVOLUTION_MODE = "schema_evolution_mode"
+    DEFAULT_SCHEMA_CONSISTENCY_TYPE = "default_schema_consistency_type"
+    SUPPORTED_READER_TYPES = "supported_reader_types"
+
+    def read_table_property(
+        table_or_table_version: Union[Table, TableVersion], property: TableProperty
+    ) -> Any:
+        properties = table_or_table_version.properties or {}
+        value = properties.get(property.value, TablePropertyDefaultValues[property])
+
+        # Handle property type conversion
+        if property == TableProperty.SUPPORTED_READER_TYPES and isinstance(value, list):
+            # Convert string values back to DatasetType enums
+            return [DatasetType(v) for v in value]
+        if property == TableProperty.SCHEMA_EVOLUTION_MODE:
+            return SchemaEvolutionMode(value)
+        if property == TableProperty.DEFAULT_SCHEMA_CONSISTENCY_TYPE:
+            return SchemaConsistencyType(value)
+        if property == TableProperty.READ_OPTIMIZATION_LEVEL:
+            return TableReadOptimizationLevel(value)
+        return value
+
+
+class TableReadOptimizationLevel(str, Enum):
+    """
+    Enum controlling the how much to optimize reads when writing to a table. Different levels
+    here correspond to different tradeoffs between write and read performance.
+
+    NONE: No read optimization. Deletes and updates are resolved by finding the values
+    that match merge key predicates by running compaction at read time. Provides the
+    fastest/cheapest writes but slow/expensive reads. Resilient to conflicts with concurrent
+    writes, including table management jobs like compaction.
+
+    MODERATE: Discover record indexes that match merge key predicates at write time and record
+    those values as logically deleted (e.g., using a bitmask). Provides faster/cheaper reads but
+    slower/more-expensive writes. May conflict with concurrent writes that remove/replace data
+    files like compaction.
+
+    MAX: Materialize all deletes and updates at write time by running compaction during
+    every write. Provides fast/cheap reads but slow/expensive writes. May conflict with
+    concurrent writes, including table management jobs like compaction.
+    """
+
+    NONE = "none"
+    MODERATE = "moderate"
+    MAX = "max"
+
+
+TablePropertyDefaultValues: Dict[TableProperty, Any] = {
+    TableProperty.READ_OPTIMIZATION_LEVEL: TableReadOptimizationLevel.MAX,
+    TableProperty.RECORDS_PER_COMPACTED_FILE: MAX_RECORDS_PER_COMPACTED_FILE,
+    TableProperty.APPENDED_RECORD_COUNT_COMPACTION_TRIGGER: MAX_RECORDS_PER_COMPACTED_FILE
+    * 2,
+    TableProperty.APPENDED_FILE_COUNT_COMPACTION_TRIGGER: 1000,
+    TableProperty.APPENDED_DELTA_COUNT_COMPACTION_TRIGGER: 100,
+    TableProperty.DEFAULT_COMPACTION_HASH_BUCKET_COUNT: 8,
+    TableProperty.SCHEMA_EVOLUTION_MODE: SchemaEvolutionMode.AUTO,
+    TableProperty.DEFAULT_SCHEMA_CONSISTENCY_TYPE: SchemaConsistencyType.NONE,
+    TableProperty.SUPPORTED_READER_TYPES: [d for d in DatasetType],
+}
 
 
-def get_table_writer(table: Union[dcs.LocalTable, dcs.DistributedDataset]) -> Callable:
-    table_writer_func = TABLE_CLASS_TO_WRITER_FUNC.get(type(table))
-    if table_writer_func is None:
+def _get_table_function(
+    table: Union[LocalTable, DistributedDataset],
+    function_map: Dict[Type, Callable],
+    operation_name: str,
+) -> Callable:
+    """Generic helper to look up table-type-specific functions."""
+    table_func = function_map.get(type(table))
+    if table_func is None:
         msg = (
-            f"No writer found for table type: {type(table)}.\n"
-            f"Known table types: {TABLE_CLASS_TO_WRITER_FUNC.keys}"
+            f"No {operation_name} function found for table type: {type(table)}.\n"
+            f"Known table types: {list(function_map.keys())}"
         )
         raise ValueError(msg)
-    return table_writer_func
+    return table_func
 
 
-def get_table_slicer(table: Union[dcs.LocalTable, dcs.DistributedDataset]) -> Callable:
-    table_slicer_func = TABLE_CLASS_TO_SLICER_FUNC.get(type(table))
-    if table_slicer_func is None:
+def _get_table_type_function(
+    table_type: DatasetType, function_map: Dict[str, Callable], operation_name: str
+) -> Callable:
+    """Generic helper to look up DatasetType-specific functions."""
+    table_func = function_map.get(table_type.value)
+    if table_func is None:
         msg = (
-            f"No slicer found for table type: {type(table)}.\n"
-            f"Known table types: {TABLE_CLASS_TO_SLICER_FUNC.keys}"
+            f"No {operation_name} function found for table type: {table_type}.\n"
+            f"Known table types: {list(function_map.keys())}"
         )
         raise ValueError(msg)
-    return table_slicer_func
+    return table_func
+
+
+def _convert_all(tables: List[LocalTable], conversion_fn: Callable, **kwargs):
+    if not tables:  # Empty list
+        return pd.DataFrame()
+
+    # Convert list elements
+    all_tables = []
+    for i, table in enumerate(tables):
+        try:
+            converted_table = conversion_fn(table, **kwargs)
+            all_tables.append(converted_table)
+        except Exception as e:
+            raise ValueError(f"Failed to convert list element {i}: {e}") from e
+
+    # Concatenate with error handling - handle different table types
+    try:
+        # Check if we have PyArrow tables
+        if all(isinstance(table, pa.Table) for table in all_tables):
+            # Use PyArrow concatenation for PyArrow tables
+            return pa.concat_tables(all_tables, promote_options="permissive")
+        else:
+            # Use pandas concatenation for other types
+            return pd.concat(all_tables, ignore_index=True, sort=False)
+    except Exception as e:
+        raise ValueError(f"Failed to concatenate {len(all_tables)} tables: {e}") from e
+
+
+def get_table_length(
+    table: Union[LocalTable, DistributedDataset, BlockAccessor]
+) -> int:
+    """
+    Generic function to get the length of a table or distributed dataset.
+
+    Args:
+        table: The local table or distributed dataset to get the length of
+
+    Returns:
+        Length of the table or distributed dataset in rows
+    """
+    # Handle DAFT DataFrames dynamically
+    if hasattr(table, "count_rows") and str(type(table).__module__).startswith("daft"):
+        return table.count_rows()
+    elif isinstance(table, RayDataset):
+        return table.count()
+    elif isinstance(table, papq.ParquetFile):
+        return table.metadata.num_rows
+    else:
+        return len(table)
+
+
+def dataset_length(table: Dataset) -> int:
+    """
+    Generic function to get the length of a dataset in records.
+    If the input is a list of tables, the length is the sum of the
+    lengths of the tables.
+
+    Args:
+        table: The dataset to get the length of
+
+    Returns:
+        Length of the dataset in records
+    """
+    if isinstance(table, list):
+        return sum(get_table_length(t) for t in table)
+    return get_table_length(table)
+
+
+def get_table_size(table: Union[LocalTable, DistributedDataset]) -> int:
+    """
+    Generic function to get the size of a table or distributed dataset.
+
+    Args:
+        table: The local table or distributed dataset to get the size of
+
+    Returns:
+        Size of the table or distributed dataset
+    """
+    table_size_func = _get_table_function(table, TABLE_CLASS_TO_SIZE_FUNC, "size")
+    return table_size_func(table)
+
+
+def dataset_size(table: Dataset) -> int:
+    """
+    Generic function to get the size of a dataset in bytes.
+    If the input is a list of tables, the size is the sum of the
+    sizes of the tables.
+
+    Args:
+        table: The dataset to get the size of
+
+    Returns:
+        Size of the dataset in bytes
+    """
+    if isinstance(table, list):
+        return sum(get_table_size(t) for t in table)
+    return get_table_size(table)
+
+
+def get_table_column_names(table: Union[LocalTable, DistributedDataset]) -> List[str]:
+    """
+    Generic function to get the column names of a table or distributed dataset.
+
+    Args:
+        table: The local table or distributed dataset to get the column names of
+
+    Returns:
+        List of column names
+    """
+    column_names_func = _get_table_function(
+        table, TABLE_CLASS_TO_COLUMN_NAMES_FUNC, "column names"
+    )
+    return column_names_func(table)
+
+
+def dataset_column_names(table: Dataset) -> List[str]:
+    """
+    Generic function to get the column names of a dataset.
+    If the input is a list of tables, unique column names are
+    returned in the order they are first seen in the list.
+
+    Args:
+        table: The dataset to get the column names of
+
+    Returns:
+        List of column names
+    """
+    if isinstance(table, list):
+        # use dictionary keys as an ordered set
+        column_names = {}
+        for t in table:
+            for column_name in get_table_column_names(t):
+                column_names[column_name] = None
+        return list(column_names.keys())
+    return get_table_column_names(table)
+
+
+def get_table_schema(table: Union[LocalTable, DistributedDataset]) -> pa.Schema:
+    """
+    Generic function to get the PyArrow schema of a table or distributed dataset.
+
+    Args:
+        table: The local table or distributed dataset to get the schema of
+
+    Returns:
+        PyArrow Schema object
+    """
+    schema_func = _get_table_function(table, TABLE_CLASS_TO_SCHEMA_FUNC, "schema")
+    return schema_func(table)
+
+
+def dataset_schema(table: Dataset) -> pa.Schema:
+    """
+    Generic function to get the PyArrow schema of a dataset. If the input is a list of
+    tables, uses pyarrow.unify_schemas(schemas, promote_options="permissive").
+
+    Args:
+        table: The dataset to get the schema of
+
+    Returns:
+        PyArrow Schema object
+    """
+    if isinstance(table, list):
+        return pa.unify_schemas(
+            [get_table_schema(t) for t in table], promote_options="permissive"
+        )
+    return get_table_schema(table)
+
+
+def get_table_writer(table: Union[LocalTable, DistributedDataset]) -> Callable:
+    """
+    Generic function to get a table writer function for a given dataset type.
+
+    Args:
+        table: The local table or distributed dataset to get the writer function for
+
+    Returns:
+        Writer function for the given dataset type
+    """
+    return _get_table_function(table, TABLE_CLASS_TO_WRITER_FUNC, "writer")
+
+
+def get_table_slicer(table: Union[LocalTable, DistributedDataset]) -> Callable:
+    """
+    Generic function to get a table slicer function for a given dataset type.
+
+    Args:
+        table: The local table or distributed dataset to get the slicer function for
+
+    Returns:
+        Slicer function for the given dataset type
+    """
+    return _get_table_function(table, TABLE_CLASS_TO_SLICER_FUNC, "slicer")
+
+
+def get_dataset_type(dataset: Dataset) -> DatasetType:
+    """Get the DatasetType enum value for a given dataset object.
+
+    Args:
+        dataset: The dataset object to identify
+
+    Returns:
+        DatasetType enum value corresponding to the dataset type
+
+    Raises:
+        ValueError: If the dataset type is not supported
+    """
+    dataset_type_str = _get_table_function(
+        dataset, TABLE_CLASS_TO_TABLE_TYPE, "dataset type identification"
+    )
+    return DatasetType(dataset_type_str)
+
+
+def table_to_pyarrow(
+    table: Union[LocalTable, DistributedDataset],
+    *,
+    schema: Optional[pa.Schema] = None,
+    **kwargs,
+) -> pa.Table:
+    """
+    Convert a single table or distributed dataset to PyArrow Table format.
+
+    Args:
+        table: The local table or distributed dataset to convert
+        schema: Optional schema to use for the conversion
+        **kwargs: Additional arguments passed to the conversion function
+
+    Returns:
+        PyArrow Table created from the provided dataset
+    """
+    to_pyarrow_func = _get_table_function(
+        table, TABLE_CLASS_TO_PYARROW_FUNC, "pyarrow conversion"
+    )
+    return to_pyarrow_func(table, schema=schema, **kwargs)
+
+
+def table_to_pandas(
+    table: Union[LocalTable, DistributedDataset],
+    *,
+    schema: Optional[pa.Schema] = None,
+    **kwargs,
+) -> pd.DataFrame:
+    """
+    Convert a single table or distributed dataset to pandas DataFrame format.
+
+    Args:
+        table: The local table or distributed dataset to convert
+        schema: Optional schema to use for the conversion
+        **kwargs: Additional arguments passed to the conversion function
+
+    Returns:
+        pandas DataFrame created from the provided dataset
+    """
+    to_pandas_func = _get_table_function(
+        table, TABLE_CLASS_TO_PANDAS_FUNC, "pandas conversion"
+    )
+    return to_pandas_func(table, schema=schema, **kwargs)
+
+
+def to_pyarrow(
+    table: Dataset, *, schema: Optional[pa.Schema] = None, **kwargs
+) -> pa.Table:
+    """
+    Convert any supported dataset type to PyArrow Table format.
+
+    Args:
+        table: The table/dataset to convert
+        schema: Optional schema to use for the conversion
+        **kwargs: Additional arguments passed to the conversion function
+
+    Returns:
+        PyArrow Table created from the provided dataset
+    """
+    if isinstance(table, list):
+        return _convert_all(table, table_to_pyarrow, schema=schema, **kwargs)
+    return table_to_pyarrow(table, schema=schema, **kwargs)
+
+
+def to_pandas(
+    table: Dataset, *, schema: Optional[pa.Schema] = None, **kwargs
+) -> pd.DataFrame:
+    """
+    Convert any supported dataset type to pandas DataFrame format.
+
+    Args:
+        table: The table/dataset to convert
+        schema: Optional schema to use for the conversion
+        **kwargs: Additional arguments passed to the conversion function
+
+    Returns:
+        pandas DataFrame created from the provided dataset
+    """
+    if isinstance(table, list):
+        return _convert_all(table, table_to_pandas, schema=schema, **kwargs)
+    return table_to_pandas(table, schema=schema, **kwargs)
+
+
+def from_pyarrow(pa_table: pa.Table, target_type: DatasetType, **kwargs) -> Dataset:
+    """Convert PyArrow Table to the specified dataset type.
+
+    Args:
+        pa_table: PyArrow Table to convert
+        target_type: Target DatasetType to convert to
+        **kwargs: Additional arguments passed to the conversion function
+
+    Returns:
+        Dataset converted to the target type
+
+    Raises:
+        ValueError: If target_type is not supported
+    """
+    conversion_func = _get_table_type_function(
+        target_type,
+        DATASET_TYPE_FROM_PYARROW,
+        f"{target_type} conversion",
+    )
+    return conversion_func(pa_table, **kwargs)
+
+
+def from_pandas(pd_df: pd.DataFrame, target_type: DatasetType, **kwargs) -> Dataset:
+    """Convert Pandas DataFrame to the specified dataset type.
+
+    Args:
+        pd_df: Pandas DataFrame to convert
+        target_type: Target DatasetType to convert to
+        **kwargs: Additional arguments passed to the conversion function
+
+    Returns:
+        Dataset converted to the target type
+
+    Raises:
+        ValueError: If target_type is not supported
+    """
+    conversion_func = _get_table_type_function(
+        target_type,
+        DATASET_TYPE_FROM_PANDAS,
+        f"{target_type} conversion",
+    )
+    return conversion_func(pd_df, **kwargs)
+
+
+def empty_table(table_type: DatasetType) -> Dataset:
+    """
+    Create an empty table of the given type.
+    """
+    empty_table_func = _get_table_type_function(
+        table_type, TABLE_TYPE_TO_EMPTY_TABLE_FUNC, "empty table"
+    )
+    return empty_table_func()
+
+
+def append_column_to_table(
+    table: LocalTable,
+    column_name: str,
+    column_value: Any,
+) -> LocalTable:
+    """
+    Generic function to append a column with a specified value to any supported dataset type.
+
+    Args:
+        table: The table/dataset to add column to
+        column_name: Name of the new column
+        column_value: Value to populate in all rows of the new column
+        table_type: Type of the dataset
+
+    Returns:
+        Updated table with the new column
+    """
+    append_column_to_table_func = _get_table_function(
+        table, TABLE_CLASS_TO_APPEND_COLUMN_FUNC, "append column"
+    )
+    return append_column_to_table_func(table, column_name, column_value)
+
+
+def select_columns_from_table(
+    table: LocalTable,
+    column_names: List[str],
+) -> LocalTable:
+    """
+    Generic function to select columns from any supported dataset type.
+
+    Args:
+        table: The table/dataset to select columns from
+        column_names: List of column names to select
+
+    Returns:
+        Updated table with the selected columns
+    """
+    select_columns_func = _get_table_function(
+        table, TABLE_CLASS_TO_SELECT_COLUMNS_FUNC, "select columns"
+    )
+    return select_columns_func(table, column_names)
+
+
+def write_sliced_table(
+    table: Union[LocalTable, DistributedDataset],
+    base_path: str,
+    filesystem: Optional[pa.fs.FileSystem],
+    max_records_per_entry: Optional[int],
+    table_writer_fn: Callable,
+    table_slicer_fn: Callable,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    content_type: ContentType = ContentType.PARQUET,
+    entry_params: Optional[EntryParams] = None,
+    entry_type: Optional[EntryType] = EntryType.DATA,
+) -> ManifestEntryList:
+    """
+    Writes table slices to 1 or more files and returns
+    manifest entries describing the uploaded files.
+
+    Args:
+        table: The local table or distributed dataset to write
+        base_path: The base path to write the table to
+        filesystem: The filesystem to write the table to
+        table_writer_fn: The function to write the table to
+        table_slicer_fn: The function to slice the table into multiple files
+        table_writer_kwargs: Additional arguments to pass to the table writer
+        content_type: The content type to write the table to
+        entry_params: Manifest entry parameters
+        entry_type: The manifest entry types to write
+
+    Returns:
+        Manifest entries describing the uploaded files
+    """
+    # @retry decorator can't be pickled by Ray, so wrap upload in Retrying
+    retrying = Retrying(
+        wait=wait_random_exponential(multiplier=1, max=60),
+        stop=stop_after_delay(UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY),
+        retry=retry_if_exception_type(RetryableError),
+    )
+
+    manifest_entries = ManifestEntryList()
+    table_record_count = get_table_length(table)
+
+    if max_records_per_entry is None or not table_record_count:
+        # write the whole table to a single file
+        manifest_entries = retrying(
+            write_table,
+            table,
+            f"{base_path}",  # cast any non-string arg to string
+            filesystem,
+            table_writer_fn,
+            table_writer_kwargs,
+            content_type,
+            entry_params,
+            entry_type,
+        )
+    else:
+        # iteratively write table slices
+        table_slices = table_slicer_fn(table, max_records_per_entry)
+        for table_slice in table_slices:
+            slice_entries = retrying(
+                write_table,
+                table_slice,
+                f"{base_path}",  # cast any non-string arg to string
+                filesystem,
+                table_writer_fn,
+                table_writer_kwargs,
+                content_type,
+                entry_params,
+                entry_type,
+            )
+            manifest_entries.extend(slice_entries)
+    return manifest_entries
+
+
+def write_table(
+    table: Union[LocalTable, DistributedDataset],
+    base_path: str,
+    filesystem: Optional[pa.fs.FileSystem],
+    table_writer_fn: Callable,
+    table_writer_kwargs: Optional[Dict[str, Any]],
+    content_type: ContentType = ContentType.PARQUET,
+    entry_params: Optional[EntryParams] = None,
+    entry_type: Optional[EntryType] = EntryType.DATA,
+) -> ManifestEntryList:
+    """
+    Writes the given table to 1 or more files and return
+    manifest entries describing the uploaded files.
+
+    Args:
+        table: The local table or distributed dataset to write
+        base_path: The base path to write the table to
+        filesystem: The filesystem to write the table to
+        table_writer_fn: The function to write the table to
+        table_writer_kwargs: Additional arguments to pass to the table writer
+        content_type: The content type to write the table to
+        entry_params: Manifest entry parameters
+        entry_type: The manifest entry types to write
+
+    Returns:
+        Manifest entries describing the uploaded files
+    """
+    if table_writer_kwargs is None:
+        table_writer_kwargs = {}
+
+    # Determine content_encoding before writing files so we can include it in filenames
+    content_encoding = None
+    if content_type in EXPLICIT_COMPRESSION_CONTENT_TYPES:
+        # TODO(pdames): Support other user-specified encodings at write time.
+        content_encoding = ContentEncoding.GZIP
+
+    wrapped_obj = (
+        CapturedBlockWritePathsActor.remote()
+        if isinstance(table, RayDataset)
+        else CapturedBlockWritePathsBase()
+    )
+    capture_object = CapturedBlockWritePaths(wrapped_obj)
+    block_write_path_provider = UuidBlockWritePathProvider(
+        capture_object,
+        base_path=base_path,
+        content_type=content_type,
+        content_encoding=content_encoding,
+    )
+    # Extract schema, schema_id, and sort_scheme_id from table_writer_kwargs
+    schema = table_writer_kwargs.pop("schema", None)
+    schema_id = table_writer_kwargs.pop("schema_id", None)
+    sort_scheme_id = table_writer_kwargs.pop("sort_scheme_id", None)
+    table_writer_fn(
+        table,
+        base_path,
+        filesystem,
+        block_write_path_provider,
+        content_type.value,
+        schema=schema,
+        **table_writer_kwargs,
+    )
+    # TODO: Add a proper fix for block_refs and write_paths not persisting in Ray actors
+    del block_write_path_provider
+    blocks = capture_object.blocks()
+    write_paths = capture_object.write_paths()
+    metadata = get_block_metadata_list(table, write_paths, blocks)
+    manifest_entries = ManifestEntryList()
+    for block_idx, path in enumerate(write_paths):
+        try:
+            manifest_entry = ManifestEntry.from_path(
+                path=path,
+                filesystem=filesystem,
+                record_count=metadata[block_idx].num_rows,
+                source_content_length=metadata[block_idx].size_bytes,
+                content_type=content_type.value,
+                content_encoding=content_encoding,
+                entry_type=entry_type,
+                entry_params=entry_params,
+                schema_id=schema_id,
+                sort_scheme_id=sort_scheme_id,
+            )
+            manifest_entries.append(manifest_entry)
+        except RETRYABLE_TRANSIENT_ERRORS as e:
+            _handle_retryable_error(e, path, "write", RetryableUploadTableError)
+        except BaseException as e:
+            _handle_non_retryable_error(
+                e,
+                path,
+                "upload",
+                NonRetryableUploadTableError,
+                f"and content_type={content_type}",
+            )
+    return manifest_entries
+
+
+@ray.remote
+class CapturedBlockWritePathsActor:
+    def __init__(self):
+        self._wrapped = CapturedBlockWritePathsBase()
+
+    def extend(self, write_paths: List[str], blocks: List[Block]) -> None:
+        self._wrapped.extend(write_paths, blocks)
+
+    def write_paths(self) -> List[str]:
+        return self._wrapped.write_paths()
+
+    def blocks(self) -> List[Block]:
+        return self._wrapped.blocks()
+
+
+class CapturedBlockWritePathsBase:
+    def __init__(self):
+        self._write_paths: List[str] = []
+        self._blocks: List[Block] = []
+
+    def extend(self, write_paths: List[str], blocks: List[Block]) -> None:
+        try:
+            iter(write_paths)
+        except TypeError:
+            pass
+        else:
+            self._write_paths.extend(write_paths)
+        try:
+            iter(blocks)
+        except TypeError:
+            pass
+        else:
+            self._blocks.extend(blocks)
+
+    def write_paths(self) -> List[str]:
+        return self._write_paths
+
+    def blocks(self) -> List[Block]:
+        return self._blocks
+
+
+class CapturedBlockWritePaths:
+    def __init__(self, wrapped=CapturedBlockWritePathsBase()):
+        self._wrapped = wrapped
+
+    def extend(self, write_paths: List[str], blocks: List[Block]) -> None:
+        return (
+            self._wrapped.extend(write_paths, blocks)
+            if isinstance(self._wrapped, CapturedBlockWritePathsBase)
+            else ray.get(self._wrapped.extend.remote(write_paths, blocks))
+        )
+
+    def write_paths(self) -> List[str]:
+        return (
+            self._wrapped.write_paths()
+            if isinstance(self._wrapped, CapturedBlockWritePathsBase)
+            else ray.get(self._wrapped.write_paths.remote())
+        )
+
+    def blocks(self) -> List[Block]:
+        return (
+            self._wrapped.blocks()
+            if isinstance(self._wrapped, CapturedBlockWritePathsBase)
+            else ray.get(self._wrapped.blocks.remote())
+        )
+
+
+class UuidBlockWritePathProvider(FilenameProvider):
+    """Block write path provider implementation that writes each
+    dataset block out to a file of the form: {base_path}/{uuid}
+    """
+
+    def __init__(
+        self,
+        capture_object: CapturedBlockWritePaths,
+        base_path: Optional[str] = None,
+        content_type: Optional[ContentType] = None,
+        content_encoding: Optional[ContentEncoding] = None,
+    ):
+        self.base_path = base_path
+        self.content_type = content_type
+        self.content_encoding = content_encoding
+        self.write_paths: List[str] = []
+        self.blocks: List[Block] = []
+        self.capture_object = capture_object
+
+    def __del__(self):
+        if self.write_paths or self.blocks:
+            self.capture_object.extend(
+                self.write_paths,
+                self.blocks,
+            )
+
+    def get_filename_for_block(
+        self,
+        block: Block,
+        task_index: int,
+        block_index: int,
+    ) -> str:
+        if self.base_path is None:
+            raise ValueError(
+                "Base path must be provided to UuidBlockWritePathProvider",
+            )
+        return self._get_write_path_for_block(
+            base_path=self.base_path,
+            block=block,
+            block_index=block_index,
+        )
+
+    def _get_write_path_for_block(
+        self,
+        base_path: str,
+        *,
+        block: Optional[Block] = None,
+        **kwargs,
+    ) -> str:
+        # Generate base UUID filename
+        filename = str(uuid4())
+
+        # Add content type extension if available
+        if self.content_type:
+            content_type_extension = None
+            content_type_extension = CONTENT_TYPE_TO_EXT.get(self.content_type)
+            if content_type_extension:
+                filename += content_type_extension
+
+        # Add content encoding extension if available
+        if self.content_encoding:
+            encoding_extension = None
+            encoding_extension = CONTENT_ENCODING_TO_EXT.get(self.content_encoding)
+            if encoding_extension:
+                filename += encoding_extension
+
+        write_path = f"{base_path}/{filename}"
+        self.write_paths.append(write_path)
+        if block is not None:
+            self.blocks.append(block)
+        return write_path
+
+    def __call__(
+        self,
+        base_path: str,
+        *,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+        dataset_uuid: Optional[str] = None,
+        block: Optional[Block] = None,
+        block_index: Optional[int] = None,
+        file_format: Optional[str] = None,
+    ) -> str:
+        return self._get_write_path_for_block(
+            base_path,
+            filesystem=filesystem,
+            dataset_uuid=dataset_uuid,
+            block=block,
+            block_index=block_index,
+            file_format=file_format,
+        )
+
+
+def get_block_metadata_list(
+    table: LocalTable,
+    write_paths: List[str],
+    blocks: List[Block],
+) -> List[BlockMetadata]:
+    """
+    Get the block metadata for a given table.
+
+    Args:
+        table: The local table or distributed dataset to get the block metadata for
+        write_paths: The list of write paths for the table
+        blocks: The list of blocks to get the metadata for
+
+    Returns:
+        List of block metadata
+    """
+    block_meta_list: List[BlockMetadata] = []
+    if not blocks:
+        # this must be a local table - ensure it was written to only 1 file
+        assert len(write_paths) == 1, (
+            f"Expected table of type '{type(table)}' to be written to 1 "
+            f"file, but found {len(write_paths)} files."
+        )
+        blocks = [table]
+    for block in blocks:
+        block_meta_list.append(get_block_metadata(block))
+    return block_meta_list
+
+
+def get_block_metadata(
+    table: Union[LocalTable, DistributedDataset, BlockAccessor],
+) -> BlockMetadata:
+    """
+    Get the block metadata for a given table.
+
+    Args:
+        table: The local table or distributed dataset to get the block metadata for
+
+    Returns:
+        Block metadata
+    """
+    table_size = None
+    table_size_func = TABLE_CLASS_TO_SIZE_FUNC.get(type(table))
+    if table_size_func:
+        table_size = table_size_func(table)
+    else:
+        logger.warning(f"Unable to estimate '{type(table)}' table size.")
+    if isinstance(table, BlockAccessor):
+        table = table.to_block()
+    return BlockMetadata(
+        num_rows=get_table_length(table),
+        size_bytes=table_size,
+        schema=None,
+        input_files=None,
+        exec_stats=None,
+    )
+
+
+def _reconstruct_manifest_entry_uri(
+    manifest_entry: ManifestEntry,
+    **kwargs,
+) -> ManifestEntry:
+    """
+    Reconstruct the full URI for a manifest entry.
+
+    Args:
+        manifest_entry: The manifest entry to reconstruct the URI for
+        **kwargs: Additional arguments to pass to the catalog properties
+
+    Returns:
+        Manifest entry with the reconstructed URI
+    """
+    # Reconstruct full URI with scheme for external readers (see GitHub issue #567)
+    from deltacat.catalog import get_catalog_properties
+
+    # Only pass kwargs that CatalogProperties actually accepts
+    catalog_kwargs = _filter_kwargs_for_catalog_properties(kwargs)
+    catalog_properties = get_catalog_properties(**catalog_kwargs)
+
+    original_uri = manifest_entry.uri
+    reconstructed_uri = catalog_properties.reconstruct_full_path(original_uri)
+    if original_uri != reconstructed_uri:
+        # Create a copy of the manifest entry with the reconstructed URI
+        reconstructed_entry = ManifestEntry(
+            uri=reconstructed_uri, url=manifest_entry.url, meta=manifest_entry.meta
+        )
+        return reconstructed_entry
+    return manifest_entry
+
+
+def _filter_kwargs_for_external_readers(kwargs: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Filter out DeltaCAT system kwargs that external file readers don't expect.
+
+    Use this when passing kwargs to external libraries like PyArrow, Pandas, Polars, etc.
+    This removes all DeltaCAT-specific parameters that would cause TypeErrors in external readers.
+
+    Args:
+        kwargs: The dictionary of arguments to filter
+
+    Returns:
+        Dictionary of arguments with DeltaCAT system kwargs removed
+    """
+    return {
+        k: v
+        for k, v in kwargs.items()
+        if k
+        not in [
+            # DeltaCAT catalog/storage system kwargs
+            "inner",
+            "catalog",
+            "ray_options_provider",
+            "distributed_dataset_type",
+            # DeltaCAT schema/reader kwargs
+            "table_version_schema",
+            "entry_params",
+            # Daft-specific kwargs
+            "io_config",
+            "ray_init_options",
+            # DeltaCAT processing kwargs
+            "column_names",
+            "include_columns",
+            "file_reader_kwargs_provider",
+            "file_path_column",
+            "max_parallelism",
+        ]
+    }
+
+
+def _filter_kwargs_for_catalog_properties(kwargs: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Filter kwargs to only include those that CatalogProperties accepts.
+
+    Use this when calling get_catalog_properties() or CatalogProperties.__init__().
+    Uses a whitelist approach - only passes known compatible parameters.
+
+    CatalogProperties.__init__ accepts: root, filesystem, storage
+    get_catalog_properties also accepts: catalog, inner
+
+    Args:
+        kwargs: The dictionary of arguments to filter
+
+    Returns:
+        Dictionary containing only CatalogProperties-compatible kwargs
+    """
+    return {
+        k: v
+        for k, v in kwargs.items()
+        if k in ["root", "filesystem", "storage", "catalog", "inner"]
+    }
+
+
+def _filter_kwargs_for_reader_functions(kwargs: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Filter kwargs for internal DeltaCAT reader functions that need most params but not catalog-specific ones.
+
+    Use this for internal DeltaCAT functions that need file reader kwargs, schema kwargs, etc.
+    but should not receive catalog/storage system parameters.
+    Preserves table_version_schema, entry_params, file reader kwargs, etc.
+
+    Args:
+        kwargs: The dictionary of arguments to filter
+
+    Returns:
+        Dictionary with catalog/storage system kwargs removed
+    """
+    return {
+        k: v
+        for k, v in kwargs.items()
+        if k
+        not in ["inner", "catalog", "ray_options_provider", "distributed_dataset_type"]
+    }
+
+
+def _extract_content_metadata(
+    manifest_entry: ManifestEntry,
+) -> Tuple[ContentType, ContentEncoding, str]:
+    """
+    Extract content type, encoding, and path from manifest entry.
+
+    Args:
+        manifest_entry: The manifest entry to extract the content metadata from
+
+    Returns:
+        Tuple of content type, encoding, and path
+    """
+    content_type = manifest_entry.meta.content_type
+    assert content_type, f"Unknown content type for manifest entry: {manifest_entry}"
+    content_type = ContentType(content_type)
+
+    content_encoding = manifest_entry.meta.content_encoding
+    assert (
+        content_encoding
+    ), f"Unknown content encoding for manifest entry: {manifest_entry}"
+    content_encoding = ContentEncoding(content_encoding)
+
+    path = manifest_entry.uri
+    if path is None:
+        path = manifest_entry.url
+
+    return content_type, content_encoding, path
+
+
+def _extract_partial_download_params(
+    manifest_entry: ManifestEntry,
+) -> Optional[PartialFileDownloadParams]:
+    """
+    Extract partial file download parameters from manifest entry.
+
+    Args:
+        manifest_entry: The manifest entry to extract the partial file download parameters from
+
+    Returns:
+        Partial file download parameters
+    """
+    if not manifest_entry.meta or not manifest_entry.meta.content_type_parameters:
+        return None
+
+    for type_params in manifest_entry.meta.content_type_parameters:
+        if isinstance(type_params, PartialFileDownloadParams):
+            return type_params
+    return None
+
+
+def _create_retry_wrapper():
+    """
+    Create a standardized Tenacity Retrying wrapper for file operations.
+
+    Returns:
+        Tenacity Retrying wrapper
+    """
+    return Retrying(
+        wait=wait_random_exponential(multiplier=1, max=60),
+        stop=stop_after_delay(DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY),
+        retry=retry_if_exception_type(RetryableError),
+    )
+
+
+def _remove_file_path_column(
+    include_columns: Optional[List[str]],
+    file_path_column: Optional[str],
+) -> Optional[List[str]]:
+    """Remove the file path system column from the include_columns list.
+
+    Args:
+        include_columns: The list of columns to include in a selection
+        file_path_column: Optional file path system column name to remove from the selection
+
+    Returns:
+        List of columns to include without the file path system column
+    """
+    if file_path_column and include_columns:
+        return [col for col in include_columns if col != file_path_column]
+    return include_columns
+
+
+def _prepare_download_arguments(
+    table_type: DatasetType,
+    column_names: Optional[List[str]],
+    include_columns: Optional[List[str]],
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider],
+    file_path_column: Optional[str],
+    **kwargs,
+) -> Dict[str, Any]:
+    """Prepare standardized arguments for download operations.
+
+    Args:
+        table_type: The type of table to download
+        column_names: The list of column names in the table
+        include_columns: The list of columns to include in the selection
+        file_reader_kwargs_provider: Optional per-content-type provider of file reader kwargs
+        file_path_column: The file path system column name
+        **kwargs: Additional arguments to pass to the file reader
+
+    Returns:
+        Dictionary of arguments for the download operation
+    """
+    reader_kwargs = _filter_kwargs_for_external_readers(kwargs)
+    processed_include_columns = _remove_file_path_column(
+        include_columns, file_path_column
+    )
+
+    return {
+        "table_type": table_type,
+        "column_names": column_names,
+        "include_columns": processed_include_columns,
+        "file_reader_kwargs_provider": file_reader_kwargs_provider,
+        "file_path_column": file_path_column,
+        **reader_kwargs,
+    }
+
+
+def _handle_retryable_error(e: Exception, path: str, operation: str, error_class: type):
+    """Handle retryable errors with standardized error message."""
+    raise error_class(
+        f"Retry {operation} for: {path} after receiving {type(e).__name__}: {e}"
+    ) from e
+
+
+def _handle_non_retryable_error(
+    e: Exception, path: str, operation: str, error_class: type, extra_context: str = ""
+):
+    """Handle non-retryable errors with logging and standardized error message."""
+    context = f" {extra_context}" if extra_context else ""
+    logger.warning(
+        f"{operation.title()} has failed for {path}{context}. Error: {e}",
+        exc_info=True,
+    )
+    raise error_class(
+        f"{operation.title()} has failed for {path}{context}: Error: {e}"
+    ) from e
+
+
+def from_manifest_table(
+    manifest_table: Union[LocalDataset, DistributedDataset],
+    dataset_type: DatasetType = DatasetType.DAFT,
+    schema: Optional[pa.Schema] = None,
+    **kwargs,
+) -> Dataset:
+    """
+    Read a manifest table (containing file paths and metadata) and download the actual data.
+
+    This utility function takes the output from a schemaless table read (which returns
+    manifest entries instead of data) and downloads the actual file contents.
+
+    Args:
+        manifest_table: Dataset containing manifest entries with file paths and metadata
+        dataset_type: The type of dataset to return (DAFT, RAY_DATASET, PYARROW, etc.)
+        schema: Optional PyArrow schema to enforce consistent column names across formats
+        **kwargs: Additional arguments forwarded to download functions
+
+    Returns:
+        Dataset containing the actual file contents
+    """
+    # Convert the manifest table to pandas for easier processing
+    # TODO(pdames): Iterate over each input manifest table in its native format
+    manifest_df = to_pandas(manifest_table)
+
+    # Reconstruct ManifestEntry objects from the manifest data
+    manifest_entries = []
+    for _, row in manifest_df.iterrows():
+        # Create ManifestMeta from the row data
+        meta = ManifestMeta.of(
+            content_length=row.get("meta_content_length"),
+            record_count=row.get("meta_record_count"),
+            content_type=row.get("meta_content_type"),
+            content_encoding=row.get("meta_content_encoding"),
+        )
+
+        # Create ManifestEntry
+        entry = ManifestEntry.of(
+            url=row["path"],
+            meta=meta,
+            mandatory=row.get("mandatory", True),
+            uuid=row.get("id"),
+        )
+        manifest_entries.append(entry)
+
+    # Create a new Manifest from the entries
+    reconstructed_manifest = Manifest.of(entries=manifest_entries)
+
+    # Add schema to kwargs if provided
+    if schema is not None:
+        kwargs["table_version_schema"] = schema
+
+    # Choose the appropriate download function based on dataset type
+    if dataset_type in DatasetType.distributed():
+        # Use distributed download function
+        # Map DatasetType to DistributedDatasetType
+        distributed_type_map = {
+            DatasetType.DAFT: DistributedDatasetType.DAFT,
+            DatasetType.RAY_DATASET: DistributedDatasetType.RAY_DATASET,
+        }
+        distributed_dataset_type = distributed_type_map.get(dataset_type)
+        if distributed_dataset_type is None:
+            raise ValueError(f"Unsupported distributed dataset type: {dataset_type}")
+
+        return download_manifest_entries_distributed(
+            manifest=reconstructed_manifest,
+            distributed_dataset_type=distributed_dataset_type,
+            **kwargs,
+        )
+    else:
+        # Use local download function
+        return download_manifest_entries(
+            manifest=reconstructed_manifest,
+            table_type=dataset_type,
+            **kwargs,
+        )
+
+
+def download_manifest_entries(
+    manifest: Manifest,
+    table_type: DatasetType = DatasetType.PYARROW,
+    max_parallelism: Optional[int] = 1,
+    column_names: Optional[List[str]] = None,
+    include_columns: Optional[List[str]] = None,
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    file_path_column: Optional[str] = None,
+    **kwargs,
+) -> LocalDataset:
+    """Download all entries in the manifest.
+
+    Args:
+        manifest: The manifest containing the entries to download
+        table_type: Dataset type to load the entries into
+        max_parallelism: Maximum parallelism to use
+        column_names: The list of column names in the table
+        include_columns: The list of columns to include in the selection
+        file_reader_kwargs_provider: Optional per-content-type provider of file reader kwargs,
+            (e.g., to pass in a custom schema for a Parquet file)
+        file_path_column: Optional file path system column name
+        **kwargs: Additional arguments to pass to the file reader
+
+    Returns:
+        Local dataset
+    """
+    if max_parallelism and max_parallelism <= 1:
+        return _download_manifest_entries(
+            manifest,
+            table_type,
+            column_names,
+            include_columns,
+            file_reader_kwargs_provider,
+            file_path_column,
+            **kwargs,
+        )
+    else:
+        return _download_manifest_entries_parallel(
+            manifest,
+            table_type,
+            max_parallelism,
+            column_names,
+            include_columns,
+            file_reader_kwargs_provider,
+            file_path_column,
+            **kwargs,
+        )
+
+
+def _download_manifest_entries(
+    manifest: Manifest,
+    table_type: DatasetType = DatasetType.PYARROW,
+    column_names: Optional[List[str]] = None,
+    include_columns: Optional[List[str]] = None,
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    file_path_column: Optional[str] = None,
+    **kwargs,
+) -> LocalDataset:
+    """Download all entries in the manifest.
+
+    Args:
+        manifest: The manifest containing the entries to download
+        table_type: Dataset type to load the entries into
+        column_names: The list of column names in the table
+        include_columns: The list of columns to include in the selection
+        file_reader_kwargs_provider: Optional per-content-type provider of file reader kwargs,
+            (e.g., to pass in a custom schema for a Parquet file)
+        file_path_column: Optional file path system column name
+        **kwargs: Additional arguments to pass to the file reader
+
+    Returns:
+        Local dataset
+    """
+    download_args = _prepare_download_arguments(
+        table_type,
+        column_names,
+        include_columns,
+        file_reader_kwargs_provider,
+        file_path_column,
+        **kwargs,
+    )
+    result = []
+    for e in manifest.entries:
+        manifest_entry = _reconstruct_manifest_entry_uri(e, **kwargs)
+        result.append(
+            download_manifest_entry(manifest_entry=manifest_entry, **download_args)
+        )
+
+    return result
+
+
+@ray.remote
+def download_manifest_entry_ray(
+    manifest_entry: ManifestEntry,
+    table_type: DatasetType = DatasetType.PYARROW,
+    column_names: Optional[List[str]] = None,
+    include_columns: Optional[List[str]] = None,
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    content_type: Optional[ContentType] = None,
+    content_encoding: Optional[ContentEncoding] = None,
+    filesystem: Optional[pyarrow.fs.FileSystem] = None,
+    file_path_column: Optional[str] = None,
+    **kwargs,
+) -> LocalTable:
+    """
+    Ray remote function for downloading manifest entries.
+
+    Args:
+        manifest_entry: The manifest entry to download
+        table_type: Dataset type to load the entry into
+        column_names: The list of column names in the table
+        include_columns: The list of columns to include in the selection
+        file_reader_kwargs_provider: Optional per-content-type provider of file reader kwargs,
+            (e.g., to pass in a custom schema for a Parquet file)
+        content_type: Optional content type of the file
+        content_encoding: Optional content encoding of the file
+        filesystem: Optional PyArrow filesystem to use to read the file
+        file_path_column: Optional file path system column name
+        **kwargs: Additional arguments to pass to the file reader
+
+    Returns:
+        Local table
+    """
+    # Make sure we normalize the table type to PyArrow to provide the correct
+    # input type to from_arrow_refs
+    effective_table_type = table_type
+    if table_type == DatasetType.RAY_DATASET:
+        effective_table_type = DatasetType.PYARROW
+
+    # Call the regular download function
+    result = download_manifest_entry(
+        manifest_entry=manifest_entry,
+        table_type=effective_table_type,
+        column_names=column_names,
+        include_columns=include_columns,
+        file_reader_kwargs_provider=file_reader_kwargs_provider,
+        content_type=content_type,
+        content_encoding=content_encoding,
+        filesystem=filesystem,
+        file_path_column=file_path_column,
+        **kwargs,
+    )
+
+    # Convert Polars DataFrame to Arrow Table for Ray dataset compatibility
+    if isinstance(result, pl.DataFrame):
+        result = result.to_arrow()
+
+    # Cast string_view columns to string to avoid cloudpickle issues
+    if isinstance(result, pa.Table):
+        result = _cast_string_view_to_string(result)
+
+    return result
+
+
+def download_manifest_entries_distributed(
+    manifest: Manifest,
+    table_type: DatasetType = DatasetType.PYARROW,
+    max_parallelism: Optional[int] = 1000,
+    column_names: Optional[List[str]] = None,
+    include_columns: Optional[List[str]] = None,
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    ray_options_provider: Callable[[int, Any], Dict[str, Any]] = None,
+    distributed_dataset_type: Optional[
+        DistributedDatasetType
+    ] = DistributedDatasetType.RAY_DATASET,
+    file_path_column: Optional[str] = None,
+    **kwargs,
+) -> DistributedDataset:
+    """Download all entries in the manifest using the given distributed dataset type.
+
+    Args:
+        manifest: The manifest containing the entries to download
+        table_type: Dataset type to load the entries into
+        max_parallelism: Maximum parallelism to use
+        column_names: The list of column names in the table
+        include_columns: The list of columns to include in the selection
+        file_reader_kwargs_provider: Optional per-content-type provider of file reader kwargs,
+            (e.g., to pass in a custom schema for a Parquet file)
+        ray_options_provider: Optional provider of Ray options
+        distributed_dataset_type: Optional distributed dataset type to use
+        file_path_column: Optional file path system column name
+        **kwargs: Additional arguments to pass to the file reader
+
+    Returns:
+        Distributed dataset
+    """
+    params = {
+        "manifest": manifest,
+        "table_type": table_type,
+        "max_parallelism": max_parallelism,
+        "column_names": column_names,
+        "include_columns": include_columns,
+        "file_reader_kwargs_provider": file_reader_kwargs_provider,
+        "ray_options_provider": ray_options_provider,
+        "file_path_column": file_path_column,
+        **kwargs,
+    }
+
+    if (
+        distributed_dataset_type
+        and distributed_dataset_type.value == DistributedDatasetType.RAY_DATASET.value
+    ):
+        result = _download_manifest_entries_ray_data_distributed(**params)
+        return result
+    elif distributed_dataset_type is not None:
+        params["distributed_dataset_type"] = distributed_dataset_type
+        return _download_manifest_entries_all_dataset_distributed(**params)
+    else:
+        raise ValueError(
+            f"Distributed dataset type {distributed_dataset_type} not supported."
+        )
+
+
+def _cast_string_view_to_string(table: pa.Table) -> pa.Table:
+    """
+    Cast any string_view columns to string type for Ray dataset compatibility.
+
+    This addresses compatibility issues where Ray datasets may have trouble with
+    string_view columns written by Polars to Feather.
+
+    Args:
+        table: PyArrow table that may contain string_view columns
+
+    Returns:
+        PyArrow table with string_view columns cast to string type
+    """
+    if not isinstance(table, pa.Table):
+        return table
+
+    schema = table.schema
+    has_string_view = False
+
+    # Check if any columns are string_view
+    for field in schema:
+        if pa.types.is_string_view(field.type):
+            has_string_view = True
+            break
+
+    if not has_string_view:
+        return table
+
+    # Convert to pandas and back to normalize string types
+    # This is a workaround since direct casting from string_view to string is not supported
+    try:
+        pandas_df = table.to_pandas()
+        # Convert back to PyArrow table, which should use regular string type
+        return pa.Table.from_pandas(pandas_df, preserve_index=False)
+    except Exception:
+        # If pandas conversion fails, return original table
+        return table
+
+
+def _download_manifest_entries_ray_data_distributed(
+    manifest: Manifest,
+    table_type: DatasetType = DatasetType.PYARROW,
+    max_parallelism: Optional[int] = 1000,
+    column_names: Optional[List[str]] = None,
+    include_columns: Optional[List[str]] = None,
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    ray_options_provider: Callable[[int, Any], Dict[str, Any]] = None,
+    file_path_column: Optional[str] = None,
+    **kwargs,
+) -> DistributedDataset:
+    """Download all entries in the manifest into a Ray dataset.
+
+    Args:
+        manifest: The manifest containing the entries to download
+        table_type: Dataset type to load the entries into
+        max_parallelism: Maximum parallelism to use
+        column_names: The list of column names in the table
+        include_columns: The list of columns to include in the selection
+        file_reader_kwargs_provider: Optional per-content-type provider of file reader kwargs,
+            (e.g., to pass in a custom schema for a Parquet file)
+        ray_options_provider: Optional provider of Ray options
+        file_path_column: Optional file path system column name
+        **kwargs: Additional arguments to pass to the file reader
+
+    Returns:
+        Ray dataset
+    """
+    table_pending_ids = []
+    manifest_entries = manifest.entries
+
+    if manifest_entries:
+        table_pending_ids = invoke_parallel(
+            manifest_entries,
+            download_manifest_entry_ray,
+            table_type,
+            column_names,
+            include_columns,
+            file_reader_kwargs_provider,
+            max_parallelism=max_parallelism,
+            options_provider=ray_options_provider,
+            file_path_column=file_path_column,
+            **kwargs,  # Pass through kwargs like include_paths
+        )
+
+    create_func = _get_table_type_function(
+        table_type, TABLE_TYPE_TO_DATASET_CREATE_FUNC_REFS, "dataset create"
+    )
+    return create_func(table_pending_ids)
+
+
+def _group_manifest_uris_by_content_type(
+    manifest: Manifest, **kwargs
+) -> Dict[Tuple[str, str], List[str]]:
+    """
+    Group manifest URIs by content type and content encoding.
+
+    Args:
+        manifest: The manifest containing the entries to group by content type
+        **kwargs: Additional arguments to pass to the catalog properties
+
+    Returns:
+        Dictionary mapping (content_type, content_encoding) tuples to lists of URIs
+    """
+    from deltacat.catalog import get_catalog_properties
+
+    catalog_properties = get_catalog_properties(**kwargs)
+
+    uris_by_type = {}
+
+    for entry in manifest.entries or []:
+        content_type = entry.meta.content_type
+        content_encoding = entry.meta.content_encoding
+        key = (content_type, content_encoding)
+
+        if key not in uris_by_type:
+            uris_by_type[key] = []
+
+        full_uri = catalog_properties.reconstruct_full_path(entry.uri)
+        uris_by_type[key].append(full_uri)
+
+    return uris_by_type
+
+
+def _download_manifest_entries_all_dataset_distributed(
+    manifest: Manifest,
+    table_type: DatasetType = DatasetType.PYARROW,
+    max_parallelism: Optional[int] = 1000,
+    column_names: Optional[List[str]] = None,
+    include_columns: Optional[List[str]] = None,
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    ray_options_provider: Callable[[int, Any], Dict[str, Any]] = None,
+    distributed_dataset_type: Optional[DatasetType] = DatasetType.DAFT,
+    file_path_column: Optional[str] = None,
+    **kwargs,
+) -> DistributedDataset:
+    """Download all entries in the manifest into a distributed dataset other than Ray Dataset.
+
+    Args:
+        manifest: The manifest containing the entries to download
+        table_type: Dataset type to load the entries into
+        max_parallelism: Maximum parallelism to use
+        column_names: The list of column names in the table
+        include_columns: The list of columns to include in the selection
+        file_reader_kwargs_provider: Optional per-content-type provider of file reader kwargs,
+            (e.g., to pass in a custom schema for a Parquet file)
+        ray_options_provider: Optional provider of Ray options
+        distributed_dataset_type: Optional distributed dataset type to use
+        file_path_column: Optional file path system column name
+        **kwargs: Additional arguments to pass to the file reader
+
+    Returns:
+        Distributed dataset
+    """
+    # Group manifest entries by content type instead of validating consistency
+    # Filter out table_version_schema from kwargs passed to catalog properties
+    filtered_kwargs = _filter_kwargs_for_catalog_properties(kwargs)
+    uris_by_content_type = _group_manifest_uris_by_content_type(
+        manifest, **filtered_kwargs
+    )
+
+    # If only one content type, use the original single-reader logic
+    if len(uris_by_content_type) == 1:
+        content_type, content_encoding = next(iter(uris_by_content_type.keys()))
+        uris = next(iter(uris_by_content_type.values()))
+
+        # Keep table_version_schema for the reader, but filter other system kwargs
+        reader_kwargs = _filter_kwargs_for_reader_functions(kwargs)
+
+        try:
+            reader_func = DISTRIBUTED_DATASET_TYPE_TO_READER_FUNC[
+                distributed_dataset_type.value
+            ]
+        except KeyError:
+            raise ValueError(
+                f"Unsupported distributed dataset type={distributed_dataset_type}. "
+                f"Supported types: {list(DISTRIBUTED_DATASET_TYPE_TO_READER_FUNC.keys())}"
+            )
+
+        return reader_func(
+            uris=uris,
+            content_type=content_type,
+            content_encoding=content_encoding,
+            column_names=column_names,
+            include_columns=include_columns,
+            read_func_kwargs_provider=file_reader_kwargs_provider,
+            ray_options_provider=ray_options_provider,
+            file_path_column=file_path_column,
+            **reader_kwargs,
+        )
+
+    # Multiple content types - read each group and union them (only for Daft)
+    if distributed_dataset_type != DistributedDatasetType.DAFT:
+        raise ValueError(
+            f"Mixed content types are only supported for Daft datasets. "
+            f"Got {len(uris_by_content_type)} different content types with dataset type {distributed_dataset_type}"
+        )
+
+    # Keep table_version_schema for the reader, but filter other system kwargs
+    reader_kwargs = _filter_kwargs_for_reader_functions(kwargs)
+
+    try:
+        reader_func = DISTRIBUTED_DATASET_TYPE_TO_READER_FUNC[
+            distributed_dataset_type.value
+        ]
+    except KeyError:
+        raise ValueError(
+            f"Unsupported distributed dataset type={distributed_dataset_type}. "
+            f"Supported types: {list(DISTRIBUTED_DATASET_TYPE_TO_READER_FUNC.keys())}"
+        )
+
+    # Read each content type group into a separate DataFrame
+    dataframes = []
+    for (content_type, content_encoding), uris in uris_by_content_type.items():
+        df = reader_func(
+            uris=uris,
+            content_type=content_type,
+            content_encoding=content_encoding,
+            column_names=column_names,
+            include_columns=include_columns,
+            read_func_kwargs_provider=file_reader_kwargs_provider,
+            ray_options_provider=ray_options_provider,
+            file_path_column=file_path_column,
+            **reader_kwargs,
+        )
+        dataframes.append(df)
+
+    # Union all DataFrames using Daft's union_all
+    if len(dataframes) == 1:
+        return dataframes[0]
+
+    result = dataframes[0]
+    for df in dataframes[1:]:
+        result = result.union_all(df)
+
+    return result
+
+
+def _download_manifest_entries_parallel(
+    manifest: Manifest,
+    table_type: DatasetType = DatasetType.PYARROW,
+    max_parallelism: Optional[int] = None,
+    column_names: Optional[List[str]] = None,
+    include_columns: Optional[List[str]] = None,
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    file_path_column: Optional[str] = None,
+    **kwargs,
+) -> LocalDataset:
+    """Download all entries in the manifest into a local dataset using multiprocessing.
+
+    Args:
+        manifest: The manifest containing the entries to download
+        table_type: Dataset type to load the entries into
+        max_parallelism: Maximum parallel processes to use for entry downloads
+        column_names: The list of column names in the table
+        include_columns: The list of columns to include in the selection
+        file_reader_kwargs_provider: Optional per-content-type provider of file reader kwargs,
+            (e.g., to pass in a custom schema for a Parquet file)
+        file_path_column: Optional file path system column name
+        **kwargs: Additional arguments to pass to the file reader
+
+    Returns:
+        Local dataset
+    """
+    download_args = _prepare_download_arguments(
+        table_type,
+        column_names,
+        include_columns,
+        file_reader_kwargs_provider,
+        file_path_column,
+        **kwargs,
+    )
+
+    entries_to_process = []
+    for e in manifest.entries:
+        manifest_entry = _reconstruct_manifest_entry_uri(e, **kwargs)
+        entries_to_process.append(manifest_entry)
+
+    tables = []
+    pool = multiprocessing.Pool(max_parallelism)
+
+    downloader = partial(download_manifest_entry, **download_args)
+    for table in pool.map(downloader, entries_to_process):
+        tables.append(table)
+    return tables
+
+
+def download_manifest_entry(
+    manifest_entry: ManifestEntry,
+    table_type: DatasetType = DatasetType.PYARROW,
+    column_names: Optional[List[str]] = None,
+    include_columns: Optional[List[str]] = None,
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    content_type: Optional[ContentType] = None,
+    content_encoding: Optional[ContentEncoding] = None,
+    filesystem: Optional[pyarrow.fs.FileSystem] = None,
+    file_path_column: Optional[str] = None,
+    **kwargs,
+) -> LocalTable:
+    """Download a single entry in the manifest into a local table.
+
+    Args:
+        manifest_entry: The manifest entry to download
+        table_type: Dataset type to load the entry into
+        column_names: The list of column names in the table
+        include_columns: The list of columns to include in the selection
+        file_reader_kwargs_provider: Optional per-content-type provider of file reader kwargs,
+            (e.g., to pass in a custom schema for a Parquet file)
+        content_type: Optional content type of the file
+        content_encoding: Optional content encoding of the file
+        filesystem: Optional PyArrow filesystem to use to read the file
+        file_path_column: Optional file path system column name
+        **kwargs: Additional arguments to pass to the file reader
+
+    Returns:
+        Local table
+    """
+    # Extract manifest metadata
+    (
+        extracted_content_type,
+        extracted_content_encoding,
+        path,
+    ) = _extract_content_metadata(manifest_entry)
+    content_type = content_type or extracted_content_type
+    content_encoding = content_encoding or extracted_content_encoding
+
+    # Extract partial download parameters
+    partial_file_download_params = _extract_partial_download_params(manifest_entry)
+
+    # Filter kwargs and process file path column
+    reader_kwargs = _filter_kwargs_for_external_readers(kwargs)
+    processed_include_columns = _remove_file_path_column(
+        include_columns, file_path_column
+    )
+
+    # Create retry wrapper and read file
+    retrying = _create_retry_wrapper()
+    table = retrying(
+        read_file,
+        path,
+        content_type,
+        content_encoding,
+        table_type,
+        column_names,
+        processed_include_columns,
+        file_reader_kwargs_provider,
+        partial_file_download_params,
+        filesystem,
+        **reader_kwargs,
+    )
+
+    # Add file path column if requested
+    if file_path_column:
+        if isinstance(table, papq.ParquetFile):
+            logger.warning(
+                f"Skipping file_path_column '{file_path_column}' for lazily materialized ParquetFile. "
+                f"File path information can be retrieved from the ParquetFile object's metadata. "
+                f"Use read_as=DatasetType.PYARROW to materialize with file path column."
+            )
+        else:
+            table = append_column_to_table(table, file_path_column, manifest_entry.uri)
+
+    return table
+
+
+@categorize_errors
+def read_file(
+    path: str,
+    content_type: ContentType,
+    content_encoding: ContentEncoding = ContentEncoding.IDENTITY,
+    table_type: DatasetType = DatasetType.PYARROW,
+    column_names: Optional[List[str]] = None,
+    include_columns: Optional[List[str]] = None,
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    partial_file_download_params: Optional[PartialFileDownloadParams] = None,
+    filesystem: Optional[pyarrow.fs.FileSystem] = None,
+    **kwargs,
+) -> LocalTable:
+    """Read a file into a local table.
+
+    Args:
+        path: The path to the file to read
+        content_type: The content type of the file
+        content_encoding: The content encoding of the file
+        table_type: Dataset type to load the file into
+        column_names: The list of column names in the table
+        include_columns: The list of columns to include in the selection
+        file_reader_kwargs_provider: Optional per-content-type provider of file reader kwargs,
+            (e.g., to pass in a custom schema for a Parquet file)
+        partial_file_download_params: Optional partial file download parameters
+        filesystem: Optional PyArrow filesystem to use to read the file
+        **kwargs: Additional arguments to pass to the file reader
+
+    Returns:
+        Local table
+    """
+    reader = TABLE_TYPE_TO_READER_FUNC[table_type.value]
+    try:
+        table = reader(
+            path,
+            content_type.value,
+            content_encoding.value,
+            filesystem,
+            column_names,
+            include_columns,
+            file_reader_kwargs_provider,
+            partial_file_download_params,
+            **kwargs,
+        )
+        return table
+    except RETRYABLE_TRANSIENT_ERRORS as e:
+        _handle_retryable_error(e, path, "download", RetryableDownloadTableError)
+    except BaseException as e:
+        _handle_non_retryable_error(
+            e,
+            path,
+            "read",
+            NonRetryableDownloadTableError,
+            f"and content_type={content_type} and encoding={content_encoding}",
+        )