deltacat 1.1.36__tar.gz → 2.0.0b2__tar.gz

deltacat-2.0.0b2/PKG-INFO

@@ -0,0 +1,47 @@
+Metadata-Version: 2.1
+Name: deltacat
+Version: 2.0.0b2
+Summary: A portable, scalable, fast, and Pythonic Data Lakehouse for AI.
+Home-page: https://github.com/ray-project/deltacat
+Author: Ray Team
+License: UNKNOWN
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Provides-Extra: iceberg
+License-File: LICENSE
+
+<p align="center">
+  <img src="media/deltacat-logo-alpha-750.png" alt="DeltaCAT Logo" style="width:55%; height:auto; text-align: center;">
+</p>
+
+DeltaCAT is a portable Pythonic Data Lakehouse powered by [Ray](https://github.com/ray-project/ray). It lets you define and manage
+fast, scalable, ACID-compliant multimodal data lakes, and has been used to [successfully manage exabyte-scale enterprise
+data lakes](https://aws.amazon.com/blogs/opensource/amazons-exabyte-scale-migration-from-apache-spark-to-ray-on-amazon-ec2/).
+
+It uses the Ray distributed compute framework together with [Apache Arrow](https://github.com/apache/arrow) and
+[Daft](https://github.com/Eventual-Inc/Daft) to efficiently scale common table management tasks, like petabyte-scale
+merge-on-read and copy-on-write operations.
+
+DeltaCAT provides four high-level components:
+1. **Catalog**: High-level APIs to create, discover, organize, share, and manage datasets.
+2. **Compute**: Distributed data management procedures to read, write, and optimize datasets.
+3. **Storage**: In-memory and on-disk multimodal dataset formats.
+4. **Sync**: Synchronize DeltaCAT datasets to data warehouses and other table formats.
+
+
+## Getting Started
+
+DeltaCAT is rapidly evolving. Usage instructions will be posted here soon!
+
+For now, feel free to peruse some of our examples:
+* https://github.com/ray-project/deltacat/tree/2.0/deltacat/examples/rivulet
+* https://github.com/ray-project/deltacat/tree/2.0/deltacat/examples/iceberg
+
+

deltacat-2.0.0b2/README.md

@@ -0,0 +1,26 @@
+<p align="center">
+  <img src="media/deltacat-logo-alpha-750.png" alt="DeltaCAT Logo" style="width:55%; height:auto; text-align: center;">
+</p>
+
+DeltaCAT is a portable Pythonic Data Lakehouse powered by [Ray](https://github.com/ray-project/ray). It lets you define and manage
+fast, scalable, ACID-compliant multimodal data lakes, and has been used to [successfully manage exabyte-scale enterprise
+data lakes](https://aws.amazon.com/blogs/opensource/amazons-exabyte-scale-migration-from-apache-spark-to-ray-on-amazon-ec2/).
+
+It uses the Ray distributed compute framework together with [Apache Arrow](https://github.com/apache/arrow) and
+[Daft](https://github.com/Eventual-Inc/Daft) to efficiently scale common table management tasks, like petabyte-scale
+merge-on-read and copy-on-write operations.
+
+DeltaCAT provides four high-level components:
+1. **Catalog**: High-level APIs to create, discover, organize, share, and manage datasets.
+2. **Compute**: Distributed data management procedures to read, write, and optimize datasets.
+3. **Storage**: In-memory and on-disk multimodal dataset formats.
+4. **Sync**: Synchronize DeltaCAT datasets to data warehouses and other table formats.
+
+
+## Getting Started
+
+DeltaCAT is rapidly evolving. Usage instructions will be posted here soon!
+
+For now, feel free to peruse some of our examples:
+* https://github.com/ray-project/deltacat/tree/2.0/deltacat/examples/rivulet
+* https://github.com/ray-project/deltacat/tree/2.0/deltacat/examples/iceberg

{deltacat-1.1.36 → deltacat-2.0.0b2}/deltacat/__init__.py

@@ -1,6 +1,12 @@
+import importlib
 import logging
 
 import deltacat.logs  # noqa: F401
+from deltacat.api import (
+    copy,
+    get,
+    put,
+)
 from deltacat.catalog.delegate import (
     alter_namespace,
     alter_table,
@@ -24,32 +30,51 @@ from deltacat.catalog.delegate import (
 from deltacat.catalog.model.catalog import (  # noqa: F401
     Catalog,
     Catalogs,
-    all_catalogs,
+    is_initialized,
     init,
+    get_catalog,
+    put_catalog,
 )
 from deltacat.catalog.model.table_definition import TableDefinition
 from deltacat.storage import (
     DistributedDataset,
+    Field,
     LifecycleState,
     ListResult,
     LocalDataset,
     LocalTable,
     Namespace,
+    PartitionKey,
+    PartitionScheme,
+    Schema,
     SchemaConsistencyType,
     SortKey,
     SortOrder,
+    SortScheme,
+    NullOrder,
 )
+from deltacat.storage.rivulet import Dataset, Datatype
 from deltacat.types.media import ContentEncoding, ContentType, TableType
 from deltacat.types.tables import TableWriteMode
 
+__iceberg__ = []
+if importlib.util.find_spec("pyiceberg") is not None:
+    from deltacat.catalog.iceberg import impl as IcebergCatalog
+
+    __iceberg__ = [
+        "IcebergCatalog",
+    ]
+
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.36"
+__version__ = "2.0.0b2"
 
 
 __all__ = [
     "__version__",
-    "all_catalogs",
+    "copy",
+    "get",
+    "put",
     "alter_table",
     "create_table",
     "drop_table",
@@ -68,20 +93,34 @@ __all__ = [
     "default_namespace",
     "write_to_table",
     "read_table",
+    "get_catalog",
+    "put_catalog",
+    "is_initialized",
     "init",
     "Catalog",
     "ContentType",
     "ContentEncoding",
     "DistributedDataset",
+    "Dataset",
+    "Datatype",
+    "Field",
+    "IcebergCatalog",
     "LifecycleState",
     "ListResult",
     "LocalDataset",
     "LocalTable",
     "Namespace",
+    "NullOrder",
+    "PartitionKey",
+    "PartitionScheme",
+    "Schema",
     "SchemaConsistencyType",
     "SortKey",
     "SortOrder",
+    "SortScheme",
     "TableDefinition",
     "TableType",
     "TableWriteMode",
 ]
+
+__all__ += __iceberg__

deltacat-2.0.0b2/deltacat/annotations.py

@@ -0,0 +1,36 @@
+def ExperimentalAPI(obj):
+    """Decorator for documenting experimental APIs.
+
+    Experimental APIs are classes and methods that are in development and may
+    change at any time in their development process. You should not expect
+    these APIs to be stable until their tag is changed to `DeveloperAPI` or
+    `PublicAPI`.
+
+    Subclasses that inherit from a ``@ExperimentalAPI`` base class can be
+    assumed experimental as well.
+
+    This decorator has no effect on runtime behavior
+    """
+    return obj
+
+
+def DeveloperAPI(obj):
+    """Decorator for documenting experimental APIs.
+
+    Developer APIs are classes and methods explicitly exposed to developers
+    for low level integrations with DeltaCAT (e.g.: compute engines, other catalogs).
+    You can generally expect these APIs to be stable sans minor changes (but less stable than public APIs).
+
+    This decorator has no effect on runtime behavior
+    """
+    return obj
+
+
+def PublicAPI(obj):
+    """Decorator for documenting public APIs.
+
+    Public APIs are classes and methods exposed to end users which are expected to remain stable across releases.
+
+    This decorator has no effect on runtime behavior
+    """
+    return obj

deltacat-2.0.0b2/deltacat/api.py

@@ -0,0 +1,168 @@
+from typing import Any
+
+
+import deltacat as dc
+from deltacat.catalog import Catalog
+
+
+def copy(source, destination):
+    src_parts = source.split("/")
+    src_parts = [part for part in src_parts if part]
+    dst_parts = destination.split("/")
+    dst_parts = [part for part in dst_parts if part]
+    if not dc.is_initialized():
+        raise ValueError("Catalog not initialized.")
+    if len(src_parts) != len(dst_parts) and len(src_parts) != len(dst_parts) + 1:
+        # TODO(pdames): Better error message.
+        raise ValueError(
+            f"Cannot copy {source} to {destination}. "
+            f"Source and destination must share the same type."
+        )
+    src_obj = get(source)
+    if len(src_parts) == 1:
+        # copy the given catalog
+        raise NotImplementedError
+    elif len(src_parts) == 2:
+        # TODO(pdames): Make catalog specification optional if there is only
+        #  one catalog (e.g., auto-retrieve src_parts[0]/dst_parts[0])
+        # copy the given namespace
+        src_namespace_name = src_parts[1]
+        dst_catalog_name = dst_parts[0]
+        dst_namespace_name = dst_parts[1] if len(dst_parts) >= 2 else src_namespace_name
+        new_namespace = dc.create_namespace(
+            namespace=dst_namespace_name,
+            properties=src_obj.properties,
+            catalog=dst_catalog_name,
+        )
+        return new_namespace
+    elif len(src_parts) == 3:
+        # copy the given table
+        raise NotImplementedError
+    elif len(src_parts) == 4:
+        # copy the given table version
+        raise NotImplementedError
+    elif len(src_parts) == 5:
+        # copy the given stream
+        raise NotImplementedError
+    elif len(src_parts) == 6:
+        # copy the given partition
+        raise NotImplementedError
+    elif len(src_parts) == 7:
+        # copy the given partition delta
+        raise NotImplementedError
+    raise ValueError(f"Invalid path: {src_parts}")
+
+
+def concat(source, destination):
+    raise NotImplementedError
+
+
+def delete(source):
+    raise NotImplementedError
+
+
+def move(source, destination):
+    raise NotImplementedError
+
+
+def list(path):
+    raise NotImplementedError
+
+
+def get(path) -> Any:
+    parts = path.split("/")
+    parts = [part for part in parts if part]
+    if not dc.is_initialized():
+        # TODO(pdames): Re-initialize DeltaCAT with all catalogs from the
+        #  last session.
+        raise ValueError("Catalog not initialized.")
+    if len(parts) == 1:
+        # TODO(pdames): Save all catalogs registered from the last session on
+        #  disk so that users don't need to re-initialize them every time.
+        # get the given catalog
+        catalog_name = parts[0]
+        return dc.get_catalog(catalog_name)
+    elif len(parts) == 2:
+        # get the given namespace
+        catalog_name = parts[0]
+        namespace_name = parts[1]
+        return dc.get_namespace(
+            namespace=namespace_name,
+            catalog=catalog_name,
+        )
+    elif len(parts) == 3:
+        # get the given table
+        raise NotImplementedError
+    elif len(parts) == 4:
+        # get the given table version
+        raise NotImplementedError
+    elif len(parts) == 5:
+        # get the given stream
+        raise NotImplementedError
+    elif len(parts) == 6:
+        # get the given partition
+        raise NotImplementedError
+    elif len(parts) == 7:
+        # get the given partition delta
+        raise NotImplementedError
+    raise ValueError(f"Invalid path: {path}")
+
+
+def put(path, *args, **kwargs) -> Any:
+    parts = path.split("/")
+    parts = [part for part in parts if part]
+    if len(parts) == 1:
+        # TODO(pdames): Save all catalogs registered from the last session on
+        #  disk so that users don't need to re-initialize them every time.
+        # register the given catalog
+        catalog_name = parts[0]
+        # Initialize default catalog using kwargs
+        catalog = Catalog(**kwargs)
+        return dc.put_catalog(catalog_name, catalog)
+    elif len(parts) == 2:
+        # register the given namespace
+        catalog_name = parts[0]
+        namespace_name = parts[1]
+        if not dc.is_initialized():
+            # TODO(pdames): Re-initialize DeltaCAT with all catalogs from the
+            #  last session.
+            raise ValueError("Catalog not initialized.")
+        new_namespace = dc.create_namespace(
+            namespace=namespace_name,
+            catalog=catalog_name,
+            *args,
+            **kwargs,
+        )
+        return new_namespace
+    elif len(parts) == 3:
+        # register the given table
+        raise NotImplementedError
+    elif len(parts) == 4:
+        # register the given table version
+        raise NotImplementedError
+    elif len(parts) == 5:
+        # register the given stream
+        raise NotImplementedError
+    elif len(parts) == 6:
+        # register the given partition
+        raise NotImplementedError
+    elif len(parts) == 7:
+        # register the given partition delta
+        raise NotImplementedError
+    raise ValueError(f"Invalid path: {path}")
+
+
+def exists(path):
+    raise NotImplementedError
+
+
+def query(path, expression):
+    raise NotImplementedError
+
+
+def tail(path):
+    raise NotImplementedError
+
+
+def head(path):
+    raise NotImplementedError

{deltacat-1.1.36 → deltacat-2.0.0b2}/deltacat/aws/s3u.py

@@ -14,7 +14,7 @@ from deltacat.aws.constants import (
     DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY,
 )
 
-import pyarrow as pa
+import pyarrow.fs
 import ray
 import s3fs
 from boto3.resources.base import ServiceResource
@@ -134,7 +134,7 @@ class UuidBlockWritePathProvider(FilenameProvider):
         self,
         base_path: str,
         *,
-        filesystem: Optional[pa.filesystem.FileSystem] = None,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
         dataset_uuid: Optional[str] = None,
         block: Optional[ObjectRef[Block]] = None,
         block_index: Optional[int] = None,
@@ -150,7 +150,7 @@ class UuidBlockWritePathProvider(FilenameProvider):
         self,
         base_path: str,
         *,
-        filesystem: Optional[pa.filesystem.FileSystem] = None,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
         dataset_uuid: Optional[str] = None,
         block: Optional[ObjectRef[Block]] = None,
         block_index: Optional[int] = None,
@@ -364,7 +364,7 @@ def upload_table(
     **s3_client_kwargs,
 ) -> ManifestEntryList:
     """
-    Writes the given table to 1 or more S3 files and return Redshift
+    Writes the given table to 1 or more S3 files and return
     manifest entries describing the uploaded files.
     """
     if s3_table_writer_kwargs is None:

deltacat-2.0.0b2/deltacat/benchmarking/benchmark_engine.py

@@ -0,0 +1,82 @@
+import sys
+import time
+from contextlib import contextmanager
+from typing import Generator, Tuple
+
+from deltacat.benchmarking.benchmark_report import BenchmarkMetric, BenchmarkStep
+from deltacat.storage.rivulet.dataset import Dataset
+from deltacat.storage.rivulet.reader.query_expression import QueryExpression
+
+
+@contextmanager
+def timed_step(description: str) -> Generator[BenchmarkStep, None, None]:
+    """Convenience for computing elapsed time of a block of code as a metric.
+
+    :param description: description of the step
+    :return: a benchmark operation populated with the elapsed time
+    """
+    metric = BenchmarkStep(description)
+    start_time = time.time()
+    yield metric
+    end_time = time.time()
+    metric.add(BenchmarkMetric("elapsed_time", 1000 * (end_time - start_time), "ms"))
+
+
+class BenchmarkEngine:
+    def __init__(self, dataset: Dataset):
+        self.dataset = dataset
+
+    def load_and_commit(
+        self, schema_name, generator, count
+    ) -> Tuple[str, BenchmarkStep]:
+        """Load count number of rows from the generator and commit.
+
+        :param generator: row generator
+        :param count: the number of rows to load into the dataset
+        :return: tuple of the manifest URI and a operation measurement
+        """
+        desc = f"load {count} from {generator}"
+        writer = self.dataset.writer(schema_name)
+        with timed_step(desc) as step:
+            rows = [generator.generate() for _ in range(count)]
+            writer.write(rows)
+            result = writer.flush()
+        step.add(BenchmarkMetric("loaded", count))
+        return result, step
+
+    def scan(self) -> Tuple[set[any], BenchmarkStep]:
+        """
+        Scans the rows in dataset and prints some basic statistics about the manifest
+
+        :return: Tuple[set[any], BenchmarkStep] - a tuple containing a set of merge keys and a benchmark step with metrics
+        """
+        keys = set()
+        object_count = 0
+        size_b = 0
+        # Note that we expect single col merge keys so we can return key set
+        # this will fail with validation error if dataset has multiple merge keys
+        merge_key_name = self.dataset.schemas["all"].get_merge_key()
+        with timed_step("full scan") as step:
+            for row in self.dataset.scan(QueryExpression()).to_pydict():
+                object_count += 1
+                size_b += sum([sys.getsizeof(x) for x in row.values()])
+                keys.add(row.get(merge_key_name))
+                # TODO replace with the actual metrics we want to measure
+        step.add(BenchmarkMetric("rows read", object_count))
+        step.add(BenchmarkMetric("size", size_b / (1024 * 1024), "MB"))
+        return keys, step
+
+    def run_queries(
+        self, description, manifest_uri, queries: list[QueryExpression]
+    ) -> BenchmarkStep:
+        object_count = 0
+        size_b = 0
+        with timed_step(description) as step:
+            for query in queries:
+                for row in self.dataset.scan(query).to_pydict():
+                    object_count += 1
+                    size_b += sum([sys.getsizeof(x) for x in row.values()])
+        # TODO replace with the actual metrics we want to measure
+        step.add(BenchmarkMetric("rows read", object_count))
+        step.add(BenchmarkMetric("size", size_b / (1024 * 1024), "MB"))
+        return step

deltacat-2.0.0b2/deltacat/benchmarking/benchmark_report.py

@@ -0,0 +1,86 @@
+from dataclasses import dataclass
+from tabulate import tabulate
+from typing import Union, Optional
+
+
+@dataclass
+class BenchmarkMetric:
+    name: str
+    value: Union[float, int]
+    unit: Optional[str] = None
+
+
+class BenchmarkStep:
+    """Captures measurements from a given operation"""
+
+    def __init__(self, description):
+        self.description: str = description
+        """Description of the operation"""
+        self._metrics: dict[str, BenchmarkMetric] = {}
+        """Description of the operation"""
+
+    def add(self, metric: BenchmarkMetric):
+        self._metrics[metric.name] = metric
+
+    def list_metrics(self):
+        """List the metrics (sorted by name)"""
+        return sorted(self._metrics.values(), key=lambda x: x.name)
+
+
+class BenchmarkRun:
+    """Class for capturing measurements for a given test suite for comparison."""
+
+    def __init__(self, suite: str, description: str):
+        self.suite = suite
+        """The test suite associated with this report."""
+        self.description = description
+        """Description of the report"""
+        self.steps: list[BenchmarkStep] = []
+        """List of steps and their metrics"""
+
+    def add(self, operation):
+        self.steps.append(operation)
+
+
+class BenchmarkReport:
+    def __init__(self, name):
+        self.name = name
+        self.runs: list[BenchmarkRun] = []
+
+    def add(self, run):
+        self.runs.append(run)
+
+    def __str__(self):
+        """Pretty-print a table that compares the metrics across each report.
+
+        We want to transpose these such that each report gets their own column and each metric gets its own row
+        (ideally grouped by operation).
+        """
+        if not self.runs:
+            print("No runs to compare!")
+            return
+        suites = set(r.suite for r in self.runs)
+        if len(suites) > 1:
+            print("Found more than one type of suite")
+            return
+        suite = self.runs[0].suite
+
+        headers = [
+            f"{suite} Operation",
+            "Metric",
+            "Unit",
+            *[r.description for r in self.runs],
+        ]
+        rows = []
+        for step_tranche in zip(*[r.steps for r in self.runs]):
+            # TODO zip by metric name instead of assuming all metrics are being measured
+            step_name = step_tranche[0].description
+            for metric_tuple in zip(*[x.list_metrics() for x in step_tranche]):
+                row = [
+                    step_name,
+                    metric_tuple[0].name,
+                    metric_tuple[0].unit,
+                    *[p.value for p in metric_tuple],
+                ]
+                rows.append(row)
+        return tabulate(rows, headers=headers, tablefmt="fancy_outline")

deltacat-2.0.0b2/deltacat/benchmarking/benchmark_suite.py

@@ -0,0 +1,11 @@
+from typing import Protocol
+
+from deltacat.benchmarking.benchmark_report import BenchmarkRun
+
+
+class BenchmarkSuite(Protocol):
+    def run(self) -> BenchmarkRun:
+        """Run the benchmark suite and produce a report.
+
+        Each report should be comparable against other reports by the same suite"""
+        ...

{deltacat-1.1.36 → deltacat-2.0.0b2}/deltacat/benchmarking/conftest.py

@@ -4,7 +4,9 @@ import pyarrow as pa
 import pyarrow.fs as pafs
 import pyarrow.parquet as papq
 import pytest
+from _pytest.terminal import TerminalReporter
 
+from deltacat.benchmarking.benchmark_report import BenchmarkReport
 from deltacat.utils.pyarrow import s3_file_to_table
 from deltacat.types.media import (
     ContentEncoding,
@@ -12,6 +14,25 @@ from deltacat.types.media import (
 )
 
 
+@pytest.fixture(autouse=True, scope="function")
+def report(request):
+    report = BenchmarkReport(request.node.name)
+
+    def final_callback():
+        terminal_reporter: TerminalReporter = request.config.pluginmanager.get_plugin(
+            "terminalreporter"
+        )
+        capture_manager = request.config.pluginmanager.get_plugin("capturemanager")
+        with capture_manager.global_and_fixture_disabled():
+            terminal_reporter.ensure_newline()
+            terminal_reporter.section(request.node.name, sep="-", blue=True, bold=True)
+            terminal_reporter.write(str(report))
+            terminal_reporter.ensure_newline()
+
+    request.addfinalizer(final_callback)
+    return report
+
+
 def pyarrow_read(path: str, columns: list[str] | None = None) -> pa.Table:
     assert path.startswith(
         "s3://"