deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/utils/url.py

@@ -0,0 +1,1325 @@
+import functools
+import json
+from typing import Callable, List, Tuple, Any, Union, Optional
+from urllib.parse import urlparse, urlunparse, parse_qs
+
+import ray
+import daft
+
+import pandas as pd
+import numpy as np
+import pyarrow as pa
+import polars as pl
+import deltacat as dc
+
+import pyarrow.csv as pacsv
+import pyarrow.json as pajson
+
+from deltacat.catalog import CatalogProperties
+from deltacat.constants import DEFAULT_NAMESPACE
+from deltacat.types.media import (
+    DatasetType,
+    DatastoreType,
+)
+from deltacat.utils import pyarrow as pa_utils
+
+from deltacat.storage import (
+    metastore,
+    Dataset,
+    Delta,
+    DeltaLocator,
+    ListResult,
+    Metafile,
+    Namespace,
+    NamespaceLocator,
+    Partition,
+    Stream,
+    StreamFormat,
+    StreamLocator,
+    PartitionLocator,
+    Table,
+    TableLocator,
+    TableVersion,
+    TableVersionLocator,
+)
+
+
+def _normalize_partition_values_from_json(partition_values):
+    """
+    Normalize partition values parsed from JSON URLs.
+
+    Both None and empty list [] represent unpartitioned data, but they should be
+    normalized to None for consistent lookup and validation.
+
+    Args:
+        partition_values: Partition values parsed from JSON
+
+    Returns:
+        None for unpartitioned data (both None and [] inputs),
+        original value for partitioned data
+    """
+    if partition_values is None or (
+        isinstance(partition_values, list) and len(partition_values) == 0
+    ):
+        return None
+    return partition_values
+
+
+RAY_DATASTORE_TYPE_TO_READER = {
+    DatastoreType.AUDIO: lambda url: functools.partial(
+        ray.data.read_audio,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.AVRO: lambda url: functools.partial(
+        ray.data.read_avro,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.BIGQUERY: lambda url: functools.partial(
+        ray.data.read_bigquery,
+        project_id=url.parsed.netloc,
+        dataset=url.path_elements[0] if url.path_elements else None,
+        **url.query_params,
+    ),
+    DatastoreType.BINARY: lambda url: functools.partial(
+        ray.data.read_binary_files,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.CSV: lambda url: functools.partial(
+        ray.data.read_csv,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.CLICKHOUSE: lambda url: functools.partial(
+        ray.data.read_clickhouse,
+        table=url.parsed.query,
+        dsn=url.url,
+        **url.query_params,
+    ),
+    DatastoreType.DATABRICKS_TABLES: lambda url: functools.partial(
+        ray.data.read_databricks_tables,
+        warehouse_id=url.parsed.netloc,
+        **url.query_params,
+    ),
+    DatastoreType.DELTA_SHARING: lambda url: functools.partial(
+        ray.data.read_delta_sharing_tables,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.HUDI: lambda url: functools.partial(
+        ray.data.read_hudi,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.ICEBERG: lambda url: functools.partial(
+        ray.data.read_iceberg,
+        table_identifier=url.parsed.netloc,
+        **url.query_params,
+    ),
+    DatastoreType.IMAGES: lambda url: functools.partial(
+        ray.data.read_images,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.JSON: lambda url: functools.partial(
+        ray.data.read_json,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.LANCE: lambda url: functools.partial(
+        ray.data.read_lance,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.MONGO: lambda url: functools.partial(
+        ray.data.read_mongo,
+        url.url,
+        **url.query_params,
+    ),
+    DatastoreType.NUMPY: lambda url: functools.partial(
+        ray.data.read_numpy,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.PARQUET: lambda url: functools.partial(
+        ray.data.read_parquet,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.TEXT: lambda url: functools.partial(
+        ray.data.read_text,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.TFRECORDS: lambda url: functools.partial(
+        ray.data.read_tfrecords,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.VIDEOS: lambda url: functools.partial(
+        ray.data.read_videos,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.WEBDATASET: lambda url: functools.partial(
+        ray.data.read_webdataset,
+        url.url_path,
+        **url.query_params,
+    ),
+}
+
+RAY_DATASTORE_TYPE_TO_WRITER = {
+    DatastoreType.BIGQUERY: lambda url: functools.partial(
+        ray.data.Dataset.write_bigquery,
+        project_id=url.parsed.netloc,
+        dataset=url.path_elements[0] if url.path_elements else None,
+        **url.query_params,
+    ),
+    DatastoreType.CSV: lambda url: functools.partial(
+        ray.data.write_csv,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.ICEBERG: lambda url: functools.partial(
+        ray.data.Dataset.write_iceberg,
+        table_identifier=url.parsed.netloc,
+        **url.query_params,
+    ),
+    DatastoreType.IMAGES: lambda url: functools.partial(
+        ray.data.Dataset.write_images,
+        path=url.url_path,
+        column=url.query_params.pop("column", "image") if url.query_params else "image",
+        **url.query_params,
+    ),
+    DatastoreType.JSON: lambda url: functools.partial(
+        ray.data.Dataset.write_json,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.LANCE: lambda url: functools.partial(
+        ray.data.Dataset.write_lance,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.MONGO: lambda url: functools.partial(
+        ray.data.Dataset.write_mongo,
+        url.url,
+        **url.query_params,
+    ),
+    DatastoreType.NUMPY: lambda url: functools.partial(
+        ray.data.Dataset.write_numpy,
+        path=url.url_path,
+        column=url.query_params.pop("column", "data") if url.query_params else "data",
+        **url.query_params,
+    ),
+    DatastoreType.PARQUET: lambda url: functools.partial(
+        ray.data.Dataset.write_parquet,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.TFRECORDS: lambda url: functools.partial(
+        ray.data.Dataset.write_tfrecords,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.WEBDATASET: lambda url: functools.partial(
+        ray.data.Dataset.write_webdataset,
+        url.url_path,
+        **url.query_params,
+    ),
+}
+
+DAFT_DATASTORE_TYPE_TO_READER = {
+    DatastoreType.CSV: lambda url: functools.partial(
+        daft.io.read_csv,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.DELTA_LAKE: lambda url: functools.partial(
+        daft.io.read_deltalake,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.HUDI: lambda url: functools.partial(
+        daft.io.read_hudi,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.ICEBERG: lambda url: functools.partial(
+        daft.io.read_iceberg,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.JSON: lambda url: functools.partial(
+        daft.io.read_json,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.PARQUET: lambda url: functools.partial(
+        daft.io.read_parquet,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.WARC: lambda url: functools.partial(
+        daft.io.read_warc,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.TEXT: lambda url: functools.partial(
+        daft.io.read_csv,
+        url.url_path,
+        infer_schema=False,
+        schema={"text": daft.DataType.string()},
+        has_headers=False,
+        delimiter=chr(25),  # end of medium char
+        double_quote=False,
+        comment=None,
+    ),
+}
+
+DAFT_DATASTORE_TYPE_TO_WRITER = {
+    DatastoreType.CSV: lambda url: functools.partial(
+        daft.DataFrame.write_csv,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.DELTA_LAKE: lambda url: functools.partial(
+        daft.DataFrame.write_deltalake,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.ICEBERG: lambda url: functools.partial(
+        daft.DataFrame.write_iceberg,
+        **url.query_params,
+    ),
+    DatastoreType.LANCE: lambda url: functools.partial(
+        daft.DataFrame.write_lance,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.PARQUET: lambda url: functools.partial(
+        daft.DataFrame.write_parquet,
+        url.url_path,
+        **url.query_params,
+    ),
+}
+
+PYARROW_DATASTORE_TYPE_TO_READER = {
+    DatastoreType.CSV: lambda url: functools.partial(
+        pa_utils.read_csv,
+        url.url_path,
+        read_options=pacsv.ReadOptions(use_threads=False),
+        **url.query_params,
+    ),
+    DatastoreType.FEATHER: lambda url: functools.partial(
+        pa_utils.read_feather,
+        url.url_path,
+        use_threads=False,
+        **url.query_params,
+    ),
+    DatastoreType.JSON: lambda url: functools.partial(
+        pa_utils.read_json,
+        url.url_path,
+        pajson.ReadOptions(use_threads=False),
+        **url.query_params,
+    ),
+    DatastoreType.ORC: lambda url: functools.partial(
+        pa_utils.read_orc,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.PARQUET: lambda url: functools.partial(
+        pa_utils.read_parquet,
+        url.url_path,
+        use_threads=False,
+        **url.query_params,
+    ),
+    DatastoreType.TEXT: lambda url: functools.partial(
+        pa_utils.read_csv,
+        url.url_path,
+        read_options=pacsv.ReadOptions(
+            use_threads=False,
+            column_names=["text"],
+        ),
+        parse_options=pacsv.ParseOptions(
+            delimiter=chr(25),  # end of medium char
+            quote_char=False,
+            double_quote=False,
+        ),
+        convert_options=pacsv.ConvertOptions(
+            check_utf8=False,
+            column_types={"text": pa.string()},
+        ),
+    ),
+}
+
+PYARROW_DATASTORE_TYPE_TO_WRITER = {
+    DatastoreType.CSV: lambda url: functools.partial(
+        pa_utils.write_csv,
+        path=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.FEATHER: lambda url: functools.partial(
+        pa_utils.write_feather,
+        path=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.ORC: lambda url: functools.partial(
+        pa_utils.write_orc,
+        path=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.PARQUET: lambda url: functools.partial(
+        pa_utils.write_parquet,
+        path=url.url_path,
+        **url.query_params,
+    ),
+}
+
+POLARS_DATASTORE_TYPE_TO_READER = {
+    DatastoreType.CSV: lambda url: functools.partial(
+        pl.read_csv,
+        url.url_path,
+        n_threads=1,
+        **url.query_params,
+    ),
+    DatastoreType.DELTA_LAKE: lambda url: functools.partial(
+        pl.read_delta,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.ICEBERG: lambda url: functools.partial(
+        pl.scan_iceberg,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.JSON: lambda url: functools.partial(
+        pl.read_json,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.PARQUET: lambda url: functools.partial(
+        pl.read_parquet,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.TEXT: lambda url: functools.partial(
+        pl.read_csv,
+        url.url_path,
+        new_columns=["text"],
+        n_threads=1,
+        separator=chr(25),  # end of medium char
+        has_header=False,
+        quote_char=None,
+        infer_schema=False,
+    ),
+}
+
+POLARS_DATASTORE_TYPE_TO_WRITER = {
+    DatastoreType.AVRO: lambda url: functools.partial(
+        pl.DataFrame.write_avro,
+        file=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.CSV: lambda url: functools.partial(
+        pl.DataFrame.write_csv,
+        file=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.DELTA_LAKE: lambda url: functools.partial(
+        pl.DataFrame.write_delta,
+        target=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.ICEBERG: lambda url: functools.partial(
+        pl.DataFrame.write_iceberg,
+        target=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.JSON: lambda url: functools.partial(
+        pl.DataFrame.write_ndjson,
+        file=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.PARQUET: lambda url: functools.partial(
+        pl.DataFrame.write_parquet,
+        file=url.url_path,
+        **url.query_params,
+    ),
+}
+
+PANDAS_DATASTORE_TYPE_TO_READER = {
+    DatastoreType.CSV: lambda url: functools.partial(
+        pd.read_csv,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.FEATHER: lambda url: functools.partial(
+        pd.read_feather,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.HDF: lambda url: functools.partial(
+        pd.read_hdf,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.HTML: lambda url: functools.partial(
+        pd.read_html,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.JSON: lambda url: functools.partial(
+        pd.read_json,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.ORC: lambda url: functools.partial(
+        pd.read_orc,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.PARQUET: lambda url: functools.partial(
+        pd.read_parquet,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.XML: lambda url: functools.partial(
+        pd.read_xml,
+        url.url_path,
+        **url.query_params,
+    ),
+}
+
+PANDAS_DATASTORE_TYPE_TO_WRITER = {
+    DatastoreType.CSV: lambda url: functools.partial(
+        pd.DataFrame.to_csv,
+        path_or_buf=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.FEATHER: lambda url: functools.partial(
+        pd.DataFrame.to_feather,
+        path=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.HDF: lambda url: functools.partial(
+        pd.DataFrame.to_hdf,
+        path_or_buf=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.HTML: lambda url: functools.partial(
+        pd.DataFrame.to_html,
+        buf=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.JSON: lambda url: functools.partial(
+        pd.DataFrame.to_json,
+        path_or_buf=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.ORC: lambda url: functools.partial(
+        pd.DataFrame.to_orc,
+        path=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.PARQUET: lambda url: functools.partial(
+        pd.DataFrame.to_parquet,
+        path=url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.XML: lambda url: functools.partial(
+        pd.DataFrame.to_xml,
+        path_or_buffer=url.url_path,
+        **url.query_params,
+    ),
+}
+
+NUMPY_DATASTORE_TYPE_TO_READER = {
+    DatastoreType.BINARY: lambda url: functools.partial(
+        np.fromfile,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.CSV: lambda url: functools.partial(
+        np.genfromtxt,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.NUMPY: lambda url: functools.partial(
+        np.load,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.TEXT: lambda url: functools.partial(
+        np.loadtxt,
+        url.url_path,
+        **url.query_params,
+    ),
+}
+
+NUMPY_DATASTORE_TYPE_TO_WRITER = {
+    DatastoreType.CSV: lambda url: functools.partial(
+        np.savetxt,
+        url.url_path,
+        delimiter=",",
+        **url.query_params,
+    ),
+    DatastoreType.NUMPY: lambda url: functools.partial(
+        np.savez_compressed,
+        url.url_path,
+        **url.query_params,
+    ),
+    DatastoreType.TEXT: lambda url: functools.partial(
+        np.savetxt,
+        url.url_path,
+        **url.query_params,
+    ),
+}
+
+DATASET_TYPE_TO_DATASTORE_TYPE_READER_RESOLVER = {
+    DatasetType.RAY_DATASET: RAY_DATASTORE_TYPE_TO_READER,
+    DatasetType.DAFT: DAFT_DATASTORE_TYPE_TO_READER,
+    DatasetType.PANDAS: PANDAS_DATASTORE_TYPE_TO_READER,
+    DatasetType.POLARS: POLARS_DATASTORE_TYPE_TO_READER,
+    DatasetType.PYARROW: PYARROW_DATASTORE_TYPE_TO_READER,
+    DatasetType.NUMPY: NUMPY_DATASTORE_TYPE_TO_READER,
+}
+
+DATASET_TYPE_TO_DATASTORE_TYPE_WRITER_RESOLVER = {
+    DatasetType.RAY_DATASET: RAY_DATASTORE_TYPE_TO_WRITER,
+    DatasetType.DAFT: DAFT_DATASTORE_TYPE_TO_WRITER,
+    DatasetType.PANDAS: PANDAS_DATASTORE_TYPE_TO_WRITER,
+    DatasetType.POLARS: POLARS_DATASTORE_TYPE_TO_WRITER,
+    DatasetType.PYARROW: PYARROW_DATASTORE_TYPE_TO_WRITER,
+    DatasetType.NUMPY: NUMPY_DATASTORE_TYPE_TO_WRITER,
+}
+
+
+class DeltaCatUrl:
+    """
+    Class for parsing DeltaCAT URLs, which are used to unambiguously locate
+    any internal object(s) already registered in a DeltaCAT catalog, or external
+    object(s) that could be registered in a DeltaCAT catalog.
+
+    Valid DeltaCAT URLs that reference internal catalog objects registered in a
+    DeltaCAT catalog include:
+
+    dc://<catalog>/[namespace]/[table]/[tableversion]/[stream]/[partition]/[delta]
+    namespace://<namespace>/[table]/[tableversion]/[stream]/[partition]/[delta]
+    table://<table>/[tableversion]/[stream]/[partition]/[delta]
+
+    Where <arg> is a required part of the URL and [arg] is an optional part of
+    the URL.
+
+    Valid DeltaCAT URLs that reference external objects include most types
+    readable into any supported DeltaCAT dataset type (e.g., Ray Data, Daft,
+    PyArrow, Pandas, Numpy). External object URLs take the form
+    <DatastoreType>+<URL> or, to be more explicit,
+    <DatastoreType>+<scheme>://<path> where `DatastoreType` is any value
+    from :class:`deltacat.types.media.DatastoreType`
+
+    To reference a file on local disk, replace <scheme>:// with "file" or
+    "local". To read an absolute local file path, use "file:///" or
+    "local:///". To read a local file path relative to the current working
+    directory, use "local://".
+
+    audio+<scheme>://<path>?param1=val1&param2=val2&...
+    avro+<scheme>://<path>?param1=val1&param2=val2&...
+    binary+<scheme>://<path>?param1=val1&param2=val2&...
+    csv+<scheme>://<path>?param1=val1&param2=val2&...
+    deltasharing+<scheme>://<path>?param1=val1&param2=val2&...
+    hudi+<scheme>://<path>?param1=val1&param2=val2&...
+    images+<scheme>://<path>?param1=val1&param2=val2&...
+    json+<scheme>://<path>?param1=val1&param2=val2&...
+    lance+<scheme>://<path>?param1=val1&param2=val2&...
+    numpy+<scheme>://<path>?param1=val1&param2=val2&...
+    parquet+<scheme>://<path>?param1=val1&param2=val2&...
+    text+<scheme>://<path>?param1=val1&param2=val2&...
+    tfrecords+<scheme>://<path>?param1=val1&param2=val2&...
+    videos+<scheme>://<path>?param1=val1&param2=val2&...
+    webdataset+<scheme>://<path>?param1=val1&param2=val2&...
+
+    Some DeltaCAT URLs reference special types of external objects
+    locatable via custom URLs that don't conform to the usual
+    <DatastoreType>+<URL> convention shown above, like:
+
+    <mongodb_uri>?database=<db_name>&collection=<collection_name>&...
+    bigquery://<project_id>/<dataset>?param1=val1&...
+    <clickhouse_dsn>?table=<table_name>?param1=val1&...
+    databricks://<warehouse_id>?param1=val1&...
+    iceberg://<table_identifier>?param1=val1&...
+
+    Note that, for reads, each of the above URLs typically resolves directly
+    to the equivalent :class:`deltacat.types.media.DatasetType` reader. For
+    example, if Ray Data is the dataset type then the equivalent
+    ray.data.read_{} API is used. In this case, a read referencing a URL of the
+    form "audio+file:///my/audio.mp4" would resolve to a call to
+    ray.data.read_audio("/my/audio.mp4").
+    """
+
+    # Auto-resolved DeltaCAT catalog path default identifiers
+    DELTACAT_URL_DEFAULT_CATALOG = "default"
+    DELTACAT_URL_DEFAULT_NAMESPACE = "default"
+    DELTACAT_URL_DEFAULT_TABLE_VERSION = "default"
+    DELTACAT_URL_DEFAULT_STREAM = "default"
+
+    def __init__(
+        self,
+        url: str,
+    ):
+        # TODO(pdames): Handle wildcard `*` and `**` at end of url.
+        self.catalog_name = None
+        self.parsed = urlparse(url, allow_fragments=False)  # support '#' in path
+        self.url = self.parsed.geturl()
+        path = self.parsed.path
+        # Remove leading/trailing slashes and split the path into elements
+        self.path_elements = [
+            element for element in path.strip("/").split("/") if path and element
+        ]
+        # Split the scheme into the root DeltaCAT scheme and the path scheme
+        self.scheme_elements = self.parsed.scheme.split("+")
+        self.datastore_type = DatastoreType(self.scheme_elements[0])
+        if len(self.scheme_elements) == 2:
+            # Remove the source/sink type from the scheme.
+            self.parsed = self.parsed._replace(scheme=self.scheme_elements[1])
+            # Save the URL path to read/write w/o the source/sink type.
+            self.url_path = urlunparse(self.parsed)
+        elif len(self.scheme_elements) > 2:
+            raise ValueError(f"Invalid DeltaCAT URL: {url}")
+        self.query_params = parse_qs(self.parsed.query) if self.parsed.query else {}
+        if self.datastore_type == DatastoreType.DELTACAT:
+            self.catalog_name = self.parsed.netloc
+            self.unresolved_namespace = (
+                self.path_elements[0] if self.path_elements else None
+            )
+            self.table = self.path_elements[1] if len(self.path_elements) > 1 else None
+            self.unresolved_table_version = (
+                self.path_elements[2] if len(self.path_elements) > 2 else None
+            )
+            self.unresolved_stream = (
+                self.path_elements[3] if len(self.path_elements) > 3 else None
+            )
+            self.partition = (
+                self.path_elements[4] if len(self.path_elements) > 4 else None
+            )
+            self.delta = self.path_elements[5] if len(self.path_elements) > 5 else None
+            self._resolve_deltacat_path_identifiers()
+        elif self.datastore_type == DatastoreType.DELTACAT_NAMESPACE:
+            self.catalog_name = DeltaCatUrl.DELTACAT_URL_DEFAULT_CATALOG
+            self.unresolved_namespace = self.parsed.netloc
+            self.table = self.path_elements[0] if self.path_elements else None
+            self.unresolved_table_version = (
+                self.path_elements[1] if len(self.path_elements) > 1 else None
+            )
+            self.unresolved_stream = (
+                self.path_elements[2] if len(self.path_elements) > 2 else None
+            )
+            self.partition = (
+                self.path_elements[3] if len(self.path_elements) > 3 else None
+            )
+            self.delta = self.path_elements[4] if len(self.path_elements) > 4 else None
+            self._resolve_deltacat_path_identifiers()
+        elif self.datastore_type == DatastoreType.DELTACAT_TABLE:
+            self.catalog_name = DeltaCatUrl.DELTACAT_URL_DEFAULT_CATALOG
+            self.unresolved_namespace = DeltaCatUrl.DELTACAT_URL_DEFAULT_NAMESPACE
+            self.table = self.parsed.netloc
+            self.unresolved_table_version = (
+                self.path_elements[0] if self.path_elements else None
+            )
+            self.unresolved_stream = (
+                self.path_elements[1] if len(self.path_elements) > 1 else None
+            )
+            self.partition = (
+                self.path_elements[2] if len(self.path_elements) > 2 else None
+            )
+            self.delta = self.path_elements[3] if len(self.path_elements) > 3 else None
+            self._resolve_deltacat_path_identifiers()
+
+    def is_deltacat_catalog_url(self):
+        return bool(self.catalog_name)
+
+    def resolve_catalog(self):
+        if self.catalog_name:
+            if self.catalog_name.lower() == DeltaCatUrl.DELTACAT_URL_DEFAULT_CATALOG:
+                self.catalog = None
+            self.catalog: CatalogProperties = dc.get_catalog(self.catalog_name).inner
+            if not isinstance(self.catalog, CatalogProperties):
+                raise ValueError(
+                    f"Expected catalog `{self.catalog_name}` to be a DeltaCAT "
+                    f"catalog but found: {self.catalog}"
+                )
+
+    def _resolve_deltacat_path_identifiers(self):
+        dc.raise_if_not_initialized()
+        self.namespace = self.table_version = self.stream = None
+        if self.unresolved_namespace:
+            if (
+                self.unresolved_namespace.lower()
+                == DeltaCatUrl.DELTACAT_URL_DEFAULT_NAMESPACE
+            ):
+                self.namespace = DEFAULT_NAMESPACE
+            else:
+                self.namespace = self.unresolved_namespace
+        if (
+            self.unresolved_table_version
+            and self.unresolved_table_version.lower()
+            != DeltaCatUrl.DELTACAT_URL_DEFAULT_TABLE_VERSION
+        ):
+            self.table_version = self.unresolved_table_version
+        if self.unresolved_stream:
+            if (
+                self.unresolved_stream.lower()
+                == DeltaCatUrl.DELTACAT_URL_DEFAULT_STREAM
+            ):
+                self.stream = StreamFormat.DELTACAT
+            else:
+                self.stream = StreamFormat(self.unresolved_stream)
+
+    def __str__(self):
+        return self.url
+
+    def __repr__(self):
+        return self.url
+
+
+def _list_table_versions(table: Table, catalog: CatalogProperties):
+    return metastore.list_table_versions(
+        namespace=table.namespace,
+        table_name=table.table_name,
+        catalog=catalog,
+    )
+
+
+def _list_streams(table_version: TableVersion, catalog: CatalogProperties):
+    return metastore.list_streams(
+        namespace=table_version.namespace,
+        table_name=table_version.table_name,
+        table_version=table_version.table_version,
+        catalog=catalog,
+    )
+
+
+class DeltaCatUrlReader:
+    def __init__(
+        self,
+        url: DeltaCatUrl,
+        dataset_type: DatasetType = DatasetType.RAY_DATASET,
+    ):
+        self._url = url
+        if url.is_deltacat_catalog_url():
+            url.resolve_catalog()
+            self._reader = DeltaCatUrlReader.resolve_dc_reader(url)
+            self._listers = DeltaCatUrlReader.resolve_dc_listers(url)
+        else:
+            self._reader = DeltaCatUrlReader.dataset_and_datastore_type_to_reader(
+                dataset_type,
+                url.datastore_type,
+            )
+
+    @property
+    def url(self) -> DeltaCatUrl:
+        return self._url
+
+    @property
+    def listers(
+        self,
+    ) -> List[
+        Tuple[
+            Callable[[Any], ListResult[Metafile]],
+            str,
+            Callable[[Metafile], Union[Metafile, str]],
+        ]
+    ]:
+        return self._listers
+
+    def read(self, *args, **kwargs) -> Dataset:
+        if self._url.is_deltacat_catalog_url():
+            return self._reader(*args, **kwargs)
+        else:
+            return self._reader(self._url)(*args, **kwargs)
+
+    @staticmethod
+    def resolve_dc_reader(url: DeltaCatUrl) -> Callable:
+        if url.delta:
+            return functools.partial(
+                metastore.get_delta,
+                namespace=url.namespace,
+                table_name=url.table,
+                table_version=url.table_version,
+                partition_values=json.loads(url.partition),
+                stream_position=url.delta,
+                catalog=url.catalog,
+            )
+        if url.partition:
+            return functools.partial(
+                metastore.get_partition,
+                stream_locator=StreamLocator.at(
+                    namespace=url.namespace,
+                    table_name=url.table,
+                    table_version=url.table_version,
+                    stream_id=None,
+                    stream_format=url.stream,
+                ),
+                partition_values=json.loads(url.partition),
+                catalog=url.catalog,
+            )
+        if url.unresolved_stream:
+            return functools.partial(
+                metastore.get_stream,
+                namespace=url.namespace,
+                table_name=url.table,
+                table_version=url.table_version,
+                stream_format=url.stream,
+                catalog=url.catalog,
+            )
+        if url.unresolved_table_version:
+            return functools.partial(
+                metastore.get_table_version,
+                namespace=url.namespace,
+                table_name=url.table,
+                table_version=url.table_version,
+                catalog=url.catalog,
+            )
+        if url.table:
+            return functools.partial(
+                metastore.get_table,
+                namespace=url.namespace,
+                table_name=url.table,
+                catalog=url.catalog,
+            )
+        if url.unresolved_namespace:
+            return functools.partial(
+                metastore.get_namespace,
+                namespace=url.namespace,
+                catalog=url.catalog,
+            )
+        if url.catalog_name:
+            return functools.partial(
+                dc.get_catalog,
+                name=url.catalog_name,
+            )
+        raise ValueError("No DeltaCAT object to read.")
+
+    @staticmethod
+    def resolve_dc_listers(
+        url: DeltaCatUrl,
+    ) -> List[
+        Tuple[
+            Callable[[Any], ListResult[Metafile]],
+            Optional[str],
+            Optional[Callable[[Metafile], Union[Metafile, str]]],
+        ]
+    ]:
+        if url.partition:
+            partition_locator = PartitionLocator.at(
+                namespace=url.namespace,
+                table_name=url.table,
+                table_version=url.table_version,
+                stream_id=None,
+                stream_format=url.stream,
+                partition_values=json.loads(url.partition),
+                partition_id=None,
+            )
+            delta_lister = functools.partial(
+                metastore.list_partition_deltas,
+                partition_like=partition_locator,
+                catalog=url.catalog,
+            )
+            return [(delta_lister, None, None)]
+        if url.unresolved_stream:
+            stream_locator = StreamLocator.at(
+                namespace=url.namespace,
+                table_name=url.table,
+                table_version=url.table_version,
+                stream_id=None,
+                stream_format=url.stream,
+            )
+            stream = Stream.of(
+                locator=stream_locator,
+                partition_scheme=None,
+            )
+            partition_lister = functools.partial(
+                metastore.list_stream_partitions,
+                stream=stream,
+                catalog=url.catalog,
+            )
+            delta_lister = functools.partial(
+                metastore.list_partition_deltas,
+                catalog=url.catalog,
+            )
+            return [
+                (partition_lister, None, None),
+                (delta_lister, "partition_like", lambda x: x),
+            ]
+        if url.unresolved_table_version:
+            stream_lister = functools.partial(
+                metastore.list_streams,
+                namespace=url.namespace,
+                table_name=url.table,
+                table_version=url.table_version,
+                catalog=url.catalog,
+            )
+            partition_lister = functools.partial(
+                metastore.list_stream_partitions,
+                catalog=url.catalog,
+            )
+            delta_lister = functools.partial(
+                metastore.list_partition_deltas,
+                catalog=url.catalog,
+            )
+            return [
+                (stream_lister, None, None),
+                (partition_lister, "stream", lambda x: x),
+                (delta_lister, "partition_like", lambda x: x),
+            ]
+        if url.table:
+            table_version_lister = functools.partial(
+                metastore.list_table_versions,
+                namespace=url.namespace,
+                table_name=url.table,
+                catalog=url.catalog,
+            )
+            stream_lister = functools.partial(
+                metastore.list_streams,
+                namespace=url.namespace,
+                table_name=url.table,
+                catalog=url.catalog,
+            )
+            partition_lister = functools.partial(
+                metastore.list_stream_partitions,
+                catalog=url.catalog,
+            )
+            delta_lister = functools.partial(
+                metastore.list_partition_deltas,
+                catalog=url.catalog,
+            )
+            return [
+                (table_version_lister, None, None),
+                (stream_lister, "table_version", lambda x: x.table_version),
+                (partition_lister, "stream", lambda x: x),
+                (delta_lister, "partition_like", lambda x: x),
+            ]
+        if url.unresolved_namespace:
+            table_lister = functools.partial(
+                metastore.list_tables,
+                namespace=url.namespace,
+                catalog=url.catalog,
+            )
+            table_version_lister = functools.partial(
+                metastore.list_table_versions,
+                namespace=url.namespace,
+                catalog=url.catalog,
+            )
+            stream_lister = functools.partial(
+                metastore.list_streams,
+                namespace=url.namespace,
+                table_name=url.table,
+                catalog=url.catalog,
+            )
+            partition_lister = functools.partial(
+                metastore.list_stream_partitions,
+                catalog=url.catalog,
+            )
+            delta_lister = functools.partial(
+                metastore.list_partition_deltas,
+                catalog=url.catalog,
+            )
+            return [
+                (table_lister, None, None),
+                (table_version_lister, "table_name", lambda x: x.table_name),
+                (stream_lister, "table_version", lambda x: x.table_version),
+                (partition_lister, "stream", lambda x: x),
+                (delta_lister, "partition_like", lambda x: x),
+            ]
+        if url.catalog_name:
+            namespace_lister = functools.partial(
+                metastore.list_namespaces,
+                catalog=url.catalog,
+            )
+            table_lister = functools.partial(
+                metastore.list_tables,
+                catalog=url.catalog,
+            )
+            table_version_lister = functools.partial(
+                _list_table_versions,
+                catalog=url.catalog,
+            )
+            stream_lister = functools.partial(
+                _list_streams,
+                catalog=url.catalog,
+            )
+            partition_lister = functools.partial(
+                metastore.list_stream_partitions,
+                catalog=url.catalog,
+            )
+            delta_lister = functools.partial(
+                metastore.list_partition_deltas,
+                catalog=url.catalog,
+            )
+            return [
+                (namespace_lister, None, None),
+                (table_lister, "namespace", lambda x: x.namespace),
+                (table_version_lister, "table", lambda x: x),
+                (stream_lister, "table_version", lambda x: x),
+                (partition_lister, "stream", lambda x: x),
+                (delta_lister, "partition_like", lambda x: x),
+            ]
+        raise ValueError("No DeltaCAT objects to list.")
+
+    @staticmethod
+    def dataset_and_datastore_type_to_reader(
+        dataset_type: DatasetType,
+        datastore_type: DatastoreType,
+    ):
+        reader_resolver = DATASET_TYPE_TO_DATASTORE_TYPE_READER_RESOLVER.get(
+            dataset_type
+        )
+        if reader_resolver is None:
+            raise ValueError(
+                f"Unsupported dataset type: {dataset_type}. "
+                f"Supported dataset types: {[dt.name for dt in DatasetType]}"
+            )
+        reader = reader_resolver.get(datastore_type)
+        if reader is None:
+            raise ValueError(
+                f"Dataset type `{dataset_type} has no reader for "
+                f"datastore type: `{datastore_type}`."
+                f"Supported datastore types: {[k.name for k in reader_resolver.keys()]}"
+            )
+        return reader
+
+
+def _stage_and_commit_stream(
+    stream: Stream,
+    *args,
+    **kwargs,
+) -> Stream:
+    """
+    Helper method to stage and commit a stream (e.g., as part of a copy
+    operation from another catalog). The committed stream will be assigned a
+    different unique ID than the input stream.
+    """
+    stream = metastore.stage_stream(
+        namespace=stream.namespace,
+        table_name=stream.table_name,
+        table_version=stream.table_version,
+        stream_format=StreamFormat(stream.stream_format),
+        *args,
+        **kwargs,
+    )
+    return metastore.commit_stream(
+        stream=stream,
+        *args,
+        **kwargs,
+    )
+
+
+def _stage_and_commit_partition(
+    partition: Partition,
+    *args,
+    **kwargs,
+) -> Partition:
+    """
+    Helper method to stage and commit a partition (e.g., as part of a copy
+    operation from another catalog). The committed partition will be assigned a
+    different unique ID than the input partition.
+    """
+    stream = metastore.get_stream(
+        namespace=partition.namespace,
+        table_name=partition.table_name,
+        table_version=partition.table_version,
+        stream_format=StreamFormat(
+            partition.stream_format or StreamFormat.DELTACAT.value
+        ),
+        *args,
+        **kwargs,
+    )
+    partition = metastore.stage_partition(
+        stream=stream,
+        partition_values=partition.partition_values,
+        partition_scheme_id=partition.partition_scheme_id,
+        *args,
+        **kwargs,
+    )
+    return metastore.commit_partition(
+        partition=partition,
+        *args,
+        **kwargs,
+    )
+
+
+class DeltaCatUrlWriter:
+    def __init__(
+        self,
+        url: DeltaCatUrl,
+        dataset_type: DatasetType = DatasetType.RAY_DATASET,
+        metafile: Optional[Metafile] = None,
+    ):
+        self._url = url
+        self._metafile = metafile
+
+        if url.is_deltacat_catalog_url():
+            if url.path_elements:
+                url.resolve_catalog()
+            self._writer = DeltaCatUrlWriter.resolve_dc_writer(url, metafile or {})
+        else:
+            self._writer = DeltaCatUrlWriter.dataset_and_datastore_type_to_writer(
+                dataset_type,
+                url.datastore_type,
+            )
+
+    @property
+    def url(self) -> DeltaCatUrl:
+        return self._url
+
+    @property
+    def metafile(self) -> Metafile:
+        return self._metafile
+
+    def write(self, suffix: str = "", *args, **kwargs) -> Union[Metafile, str]:
+        if self._url.is_deltacat_catalog_url():
+            return self._writer(*args, **kwargs)
+        else:
+            dest_url = DeltaCatUrl(f"{self._url.url}{suffix}")
+            self._writer(dest_url)(*args, **kwargs)
+            return dest_url.url_path
+
+    @staticmethod
+    def resolve_dc_writer(
+        url: DeltaCatUrl,
+        metafile: Metafile,
+    ) -> Callable:
+        if url.delta:
+            delta: Delta = Delta(
+                Metafile.based_on(
+                    other=metafile,
+                    new_id=url.delta,
+                )
+            )
+            delta.locator = DeltaLocator.at(
+                namespace=url.namespace,
+                table_name=url.table,
+                table_version=url.table_version,
+                stream_id=None,
+                stream_format=url.stream,
+                partition_values=json.loads(url.partition),
+                partition_id=None,
+                stream_position=int(url.delta),
+            )
+            # TODO(pdames): Honor deep vs. shallow copies. Deep copies require
+            #  first ensuring that all files in the source delta manifest are
+            #  staged to the target catalog before commit. For deltas whose
+            #  manifests reference local files, shallow delta copies may be
+            #  invalid in the target catalog, and should be blocked or
+            #  converted to a deep copy automatically.
+            return functools.partial(
+                metastore.commit_delta,
+                delta=delta,
+                catalog=url.catalog,
+            )
+        if url.partition:
+            partition: Partition = Partition(metafile)
+            partition.locator = PartitionLocator.at(
+                namespace=url.namespace,
+                table_name=url.table,
+                table_version=url.table_version,
+                stream_id=None,
+                stream_format=url.stream,
+                partition_values=json.loads(url.partition),
+                partition_id=None,
+            )
+            return functools.partial(
+                _stage_and_commit_partition,
+                partition=partition,
+                catalog=url.catalog,
+            )
+        if url.unresolved_stream:
+            stream: Stream = Stream(metafile)
+            stream.locator = StreamLocator.at(
+                namespace=url.namespace,
+                table_name=url.table,
+                table_version=url.table_version,
+                stream_id=None,
+                stream_format=url.stream,
+            )
+            return functools.partial(
+                _stage_and_commit_stream,
+                stream=stream,
+                catalog=url.catalog,
+            )
+        if url.unresolved_table_version:
+            table_version: TableVersion = TableVersion(metafile)
+            table_version.locator = TableVersionLocator.at(
+                namespace=url.namespace,
+                table_name=url.table,
+                table_version=url.table_version,
+            )
+            return functools.partial(
+                metastore.create_table_version,
+                namespace=table_version.namespace,
+                table_name=table_version.table_name,
+                table_version=table_version.table_version,
+                lifecycle_state=table_version.state,
+                schema=table_version.schema,
+                partition_scheme=table_version.partition_scheme,
+                sort_keys=table_version.sort_scheme,
+                table_version_description=table_version.description,
+                table_version_properties=table_version.properties,
+                supported_content_types=table_version.content_types,
+                catalog=url.catalog,
+            )
+        if url.table:
+            table: Table = Table(metafile)
+            table.locator = TableLocator.at(
+                namespace=url.namespace,
+                table_name=url.table,
+            )
+            return functools.partial(
+                metastore.create_table,
+                namespace=table.namespace,
+                table_name=table.table_name,
+                description=table.description,
+                properties=table.properties,
+                catalog=url.catalog,
+            )
+        if url.unresolved_namespace:
+            namespace: Namespace = Namespace(metafile)
+            namespace.locator = NamespaceLocator.of(
+                namespace=url.namespace,
+            )
+            return functools.partial(
+                metastore.create_namespace,
+                namespace=url.namespace,
+                properties=namespace.properties,
+                catalog=url.catalog,
+            )
+        if url.catalog_name:
+            return functools.partial(
+                dc.put_catalog,
+                name=url.catalog_name,
+            )
+        raise ValueError("No DeltaCAT object to write.")
+
+    @staticmethod
+    def dataset_and_datastore_type_to_writer(
+        dataset_type: DatasetType,
+        datastore_type: DatastoreType,
+    ):
+        writer_resolver = DATASET_TYPE_TO_DATASTORE_TYPE_WRITER_RESOLVER.get(
+            dataset_type
+        )
+        if writer_resolver is None:
+            raise ValueError(
+                f"Unsupported dataset type: {dataset_type}. "
+                f"Supported dataset types: {[dt.name for dt in DatasetType]}"
+            )
+        writer = writer_resolver.get(datastore_type)
+        if writer is None:
+            raise ValueError(
+                f"Dataset type `{dataset_type} has no writer for "
+                f"datastore type: `{datastore_type}`."
+                f"Supported datastore types: {[k.name for k in writer_resolver.keys()]}"
+            )
+        return writer