deltacat 1.1.36__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

deltacat/benchmarking/test_benchmark_pipeline.py

@@ -0,0 +1,106 @@
+import math
+from random import shuffle
+import pytest
+from deltacat.storage.rivulet.dataset import Dataset
+from deltacat.storage.rivulet.schema.datatype import Datatype
+from deltacat.storage.rivulet.reader.query_expression import QueryExpression
+from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.benchmarking.benchmark_engine import BenchmarkEngine
+from deltacat.benchmarking.benchmark_report import BenchmarkRun, BenchmarkReport
+from deltacat.benchmarking.benchmark_suite import BenchmarkSuite
+from deltacat.benchmarking.data.random_row_generator import RandomRowGenerator
+from deltacat.benchmarking.data.row_generator import RowGenerator
+from deltacat.tests.test_utils.filesystem import temp_dir_autocleanup
+
+pytestmark = pytest.mark.benchmark
+
+
+@pytest.fixture
+def schema():
+    return Schema(
+        [
+            ("id", Datatype.int32()),
+            ("source", Datatype.string()),
+            ("media", Datatype.image("png")),
+        ],
+        "id",
+    )
+
+
+class LoadAndScanSuite(BenchmarkSuite):
+    """Load some number of rows and scan"""
+
+    schema_name = "LoadAndScanSuite"
+
+    def __init__(self, dataset: Dataset, schema: Schema, generator, description=None):
+        self.suite = "ReadSuite"
+        self.dataset: Dataset = dataset
+        self.schema = schema
+        self.dataset.add_schema(schema, LoadAndScanSuite.schema_name)
+        self.generator: RowGenerator = generator
+        self.description: str = description or f"{self.dataset} x {self.generator}"
+
+    def run(self) -> BenchmarkRun:
+        container = BenchmarkEngine(self.dataset)
+        run = BenchmarkRun(self.suite, self.description)
+        # load a large number of rows
+        manifest_uri, step = container.load_and_commit(
+            LoadAndScanSuite.schema_name, self.generator, 1000
+        )
+        run.add(step)
+        # do a full scan of all rows (and eagerly load them)
+        keys, step = container.scan()
+        run.add(step)
+        # randomly retrieve all keys one-by-one from the dataset
+        random_keys = list(keys)
+        shuffle(random_keys)
+        step = container.run_queries(
+            "load all keys individually",
+            manifest_uri,
+            [QueryExpression().with_key(k) for k in random_keys],
+        )
+        run.add(step)
+        # split into 4 key ranges and get them individually
+        quartiles = self._generate_quartiles(keys)
+        expressions = [
+            QueryExpression().with_range(start, end) for (start, end) in quartiles
+        ]
+        step = container.run_queries(
+            "load key ranges by quartile", manifest_uri, expressions
+        )
+        run.add(step)
+        return run
+
+    @staticmethod
+    def _generate_quartiles(keys):
+        sorted_keys = sorted(keys)
+        size = len(keys)
+        starts = list(range(0, size, math.ceil(size / 4)))
+        ends = list([x - 1 for x in starts[1:]])
+        ends.append(size - 1)
+        quartiles = list(zip(starts, ends))
+        return [(sorted_keys[start], sorted_keys[end]) for (start, end) in quartiles]
+
+
+def test_suite1(schema: Schema, report: BenchmarkReport):
+    with temp_dir_autocleanup() as temp_dir:
+        generator = RandomRowGenerator(123, temp_dir)
+        report.add(
+            LoadAndScanSuite(
+                Dataset(dataset_name="test_suite1_ds1", metadata_uri=temp_dir),
+                schema,
+                generator,
+                "SST (rand)",
+            ).run()
+        )
+
+    with temp_dir_autocleanup() as temp_dir:
+        generator = RandomRowGenerator(123, temp_dir)
+        report.add(
+            LoadAndScanSuite(
+                Dataset(dataset_name="test_suite1_ds2", metadata_uri=temp_dir),
+                schema,
+                generator,
+                "dupe",
+            ).run()
+        )

deltacat/catalog/__init__.py

@@ -0,0 +1,14 @@
+from deltacat.catalog.model.properties import (  # noqa: F401
+    CatalogProperties,
+    get_catalog_properties,
+)
+from deltacat.catalog.model.catalog import Catalog, Catalogs  # noqa: F401
+from deltacat.catalog.main import impl as DeltacatCatalog
+
+__all__ = [
+    "CatalogProperties",
+    "get_catalog_properties",
+    "Catalog",
+    "Catalogs",
+    "DeltacatCatalog",
+]

deltacat/catalog/delegate.py

@@ -1,53 +1,33 @@
-from typing import Any, Dict, List, Optional, Set, Union
+from typing import Any, Dict, List, Optional, Union
 
-import pyarrow as pa
-import ray
-
-from deltacat.catalog.model.catalog import Catalog, all_catalogs
+from deltacat.catalog.model.catalog import get_catalog
 from deltacat.catalog.model.table_definition import TableDefinition
-from deltacat.storage.model.sort_key import SortKey
+from deltacat.storage.model.partition import PartitionScheme
+from deltacat.storage.model.sort_key import SortScheme
 from deltacat.storage.model.list_result import ListResult
-from deltacat.storage.model.namespace import Namespace
+from deltacat.storage.model.namespace import Namespace, NamespaceProperties
+from deltacat.storage.model.schema import Schema
+from deltacat.storage.model.table import TableProperties
 from deltacat.storage.model.types import (
     DistributedDataset,
     LifecycleState,
     LocalDataset,
     LocalTable,
-    SchemaConsistencyType,
+    StreamFormat,
 )
 from deltacat.types.media import ContentType
 from deltacat.types.tables import TableWriteMode
 
 
-def _get_catalog(name: Optional[str] = None) -> Catalog:
-    if not all_catalogs:
-        raise ValueError(
-            "No catalogs available! Call "
-            "`deltacat.init(catalogs={...})` to register one or more "
-            "catalogs then retry."
-        )
-    catalog = (
-        ray.get(all_catalogs.get.remote(name))
-        if name
-        else ray.get(all_catalogs.default.remote())
-    )
-    if not catalog:
-        available_catalogs = ray.get(all_catalogs.all.remote()).values()
-        raise ValueError(
-            f"Catalog '{name}' not found. Available catalogs: " f"{available_catalogs}."
-        )
-    return catalog
-
-
 # table functions
 def write_to_table(
     data: Union[LocalTable, LocalDataset, DistributedDataset],
     table: str,
+    *args,
     namespace: Optional[str] = None,
     catalog: Optional[str] = None,
     mode: TableWriteMode = TableWriteMode.AUTO,
     content_type: ContentType = ContentType.PARQUET,
-    *args,
     **kwargs,
 ) -> None:
     """Write local or distributed data to a table. Raises an error if the
@@ -57,228 +37,341 @@ def write_to_table(
     specified as additional keyword arguments. When appending to, or replacing,
     an existing table, all `alter_table` parameters may be optionally specified
     as additional keyword arguments."""
-    _get_catalog(catalog).impl.write_to_table(
-        data, table, namespace, mode, content_type, *args, **kwargs
+    catalog_obj = get_catalog(catalog)
+    catalog_obj.impl.write_to_table(
+        data,
+        table,
+        *args,
+        namespace=namespace,
+        mode=mode,
+        content_type=content_type,
+        inner=catalog_obj.inner,
+        **kwargs,
     )
 
 
 def read_table(
     table: str,
+    *args,
     namespace: Optional[str] = None,
     catalog: Optional[str] = None,
-    *args,
     **kwargs,
 ) -> DistributedDataset:
     """Read a table into a distributed dataset."""
-    return _get_catalog(catalog).impl.read_table(table, namespace, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    return catalog_obj.impl.read_table(
+        table,
+        *args,
+        namespace=namespace,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 def alter_table(
     table: str,
+    *args,
     namespace: Optional[str] = None,
     catalog: Optional[str] = None,
     lifecycle_state: Optional[LifecycleState] = None,
     schema_updates: Optional[Dict[str, Any]] = None,
     partition_updates: Optional[Dict[str, Any]] = None,
-    primary_keys: Optional[Set[str]] = None,
-    sort_keys: Optional[List[SortKey]] = None,
+    sort_keys: Optional[SortScheme] = None,
     description: Optional[str] = None,
-    properties: Optional[Dict[str, str]] = None,
-    *args,
+    properties: Optional[TableProperties] = None,
     **kwargs,
 ) -> None:
     """Alter table definition."""
-    _get_catalog(catalog).impl.alter_table(
+    catalog_obj = get_catalog(catalog)
+    catalog_obj.impl.alter_table(
         table,
-        namespace,
-        lifecycle_state,
-        schema_updates,
-        partition_updates,
-        primary_keys,
-        sort_keys,
-        description,
-        properties,
         *args,
+        namespace=namespace,
+        lifecycle_state=lifecycle_state,
+        schema_updates=schema_updates,
+        partition_updates=partition_updates,
+        sort_keys=sort_keys,
+        description=description,
+        properties=properties,
+        inner=catalog_obj.inner,
         **kwargs,
     )
 
 
 def create_table(
-    table: str,
+    name: str,
+    *args,
     namespace: Optional[str] = None,
     catalog: Optional[str] = None,
-    lifecycle_state: Optional[LifecycleState] = None,
-    schema: Optional[Union[pa.Schema, str, bytes]] = None,
-    schema_consistency: Optional[Dict[str, SchemaConsistencyType]] = None,
-    partition_keys: Optional[List[Dict[str, Any]]] = None,
-    primary_keys: Optional[Set[str]] = None,
-    sort_keys: Optional[List[SortKey]] = None,
+    version: Optional[str] = None,
+    lifecycle_state: Optional[LifecycleState] = LifecycleState.ACTIVE,
+    schema: Optional[Schema] = None,
+    partition_scheme: Optional[PartitionScheme] = None,
+    sort_keys: Optional[SortScheme] = None,
     description: Optional[str] = None,
-    properties: Optional[Dict[str, str]] = None,
-    permissions: Optional[Dict[str, Any]] = None,
+    table_properties: Optional[TableProperties] = None,
+    namespace_properties: Optional[NamespaceProperties] = None,
     content_types: Optional[List[ContentType]] = None,
-    replace_existing_table: bool = False,
-    *args,
+    fail_if_exists: bool = True,
     **kwargs,
 ) -> TableDefinition:
     """Create an empty table. Raises an error if the table already exists and
-    `replace_existing_table` is False."""
-    return _get_catalog(catalog).impl.create_table(
-        table,
-        namespace,
-        lifecycle_state,
-        schema,
-        schema_consistency,
-        partition_keys,
-        primary_keys,
-        sort_keys,
-        description,
-        properties,
-        permissions,
-        content_types,
-        replace_existing_table,
+    `fail_if_exists` is True (default behavior)."""
+    catalog_obj = get_catalog(catalog)
+    return catalog_obj.impl.create_table(
+        name,
         *args,
+        namespace=namespace,
+        version=version,
+        lifecycle_state=lifecycle_state,
+        schema=schema,
+        partition_scheme=partition_scheme,
+        sort_keys=sort_keys,
+        description=description,
+        table_properties=table_properties,
+        namespace_properties=namespace_properties,
+        content_types=content_types,
+        fail_if_exists=fail_if_exists,
+        inner=catalog_obj.inner,
         **kwargs,
     )
 
 
 def drop_table(
-    table: str,
+    name: str,
+    *args,
     namespace: Optional[str] = None,
     catalog: Optional[str] = None,
+    table_version: Optional[str] = None,
     purge: bool = False,
-    *args,
     **kwargs,
 ) -> None:
     """Drop a table from the catalog and optionally purge it. Raises an error
     if the table does not exist."""
-    _get_catalog(catalog).impl.drop_table(table, namespace, purge, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    catalog_obj.impl.drop_table(
+        name,
+        *args,
+        namespace=namespace,
+        table_version=table_version,
+        purge=purge,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 def refresh_table(
     table: str,
+    *args,
     namespace: Optional[str] = None,
     catalog: Optional[str] = None,
-    *args,
     **kwargs,
 ) -> None:
     """Refresh metadata cached on the Ray cluster for the given table."""
-    _get_catalog(catalog).impl.refresh_table(table, namespace, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    catalog_obj.impl.refresh_table(
+        table,
+        *args,
+        namespace=namespace,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 def list_tables(
-    namespace: Optional[str] = None, catalog: Optional[str] = None, *args, **kwargs
+    *args, namespace: Optional[str] = None, catalog: Optional[str] = None, **kwargs
 ) -> ListResult[TableDefinition]:
     """List a page of table definitions. Raises an error if the given namespace
     does not exist."""
-    return _get_catalog(catalog).impl.list_tables(namespace, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    return catalog_obj.impl.list_tables(
+        *args,
+        namespace=namespace,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 def get_table(
-    table: str,
+    name: str,
+    *args,
     namespace: Optional[str] = None,
     catalog: Optional[str] = None,
-    *args,
+    table_version: Optional[str] = None,
+    stream_format: StreamFormat = StreamFormat.DELTACAT,
     **kwargs,
 ) -> Optional[TableDefinition]:
     """Get table definition metadata. Returns None if the given table does not
     exist."""
-    return _get_catalog(catalog).impl.get_table(table, namespace, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    return catalog_obj.impl.get_table(
+        name,
+        *args,
+        namespace=namespace,
+        table_version=table_version,
+        stream_format=stream_format,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 def truncate_table(
     table: str,
+    *args,
     namespace: Optional[str] = None,
     catalog: Optional[str] = None,
-    *args,
     **kwargs,
 ) -> None:
     """Truncate table data. Raises an error if the table does not exist."""
-    _get_catalog(catalog).impl.truncate_table(table, namespace, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    catalog_obj.impl.truncate_table(
+        table,
+        *args,
+        namespace=namespace,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 def rename_table(
     table: str,
     new_name: str,
+    *args,
     namespace: Optional[str] = None,
     catalog: Optional[str] = None,
-    *args,
     **kwargs,
 ) -> None:
     """Rename a table."""
-    _get_catalog(catalog).impl.rename_table(table, new_name, namespace, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    catalog_obj.impl.rename_table(
+        table,
+        new_name,
+        *args,
+        namespace=namespace,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 def table_exists(
     table: str,
+    *args,
     namespace: Optional[str] = None,
     catalog: Optional[str] = None,
-    *args,
     **kwargs,
 ) -> bool:
     """Returns True if the given table exists, False if not."""
-    return _get_catalog(catalog).impl.table_exists(table, namespace, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    return catalog_obj.impl.table_exists(
+        table,
+        *args,
+        namespace=namespace,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 # namespace functions
 def list_namespaces(
-    catalog: Optional[str] = None, *args, **kwargs
+    *args, catalog: Optional[str] = None, **kwargs
 ) -> ListResult[Namespace]:
     """List a page of table namespaces."""
-    return _get_catalog(catalog).impl.list_namespaces(*args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    return catalog_obj.impl.list_namespaces(
+        *args,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 def get_namespace(
-    namespace: str, catalog: Optional[str] = None, *args, **kwargs
+    namespace: str,
+    catalog: Optional[str] = None,
+    *args,
+    **kwargs,
 ) -> Optional[Namespace]:
     """Get table namespace metadata for the specified table namespace. Returns
     None if the given namespace does not exist."""
-    return _get_catalog(catalog).impl.get_namespace(namespace, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    return catalog_obj.impl.get_namespace(
+        namespace,
+        *args,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 def namespace_exists(
-    namespace: str, catalog: Optional[str] = None, *args, **kwargs
+    namespace: str,
+    catalog: Optional[str] = None,
+    *args,
+    **kwargs,
 ) -> bool:
     """Returns True if the given table namespace exists, False if not."""
-    return _get_catalog(catalog).impl.namespace_exists(namespace, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    return catalog_obj.impl.namespace_exists(
+        namespace,
+        *args,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
 def create_namespace(
     namespace: str,
-    permissions: Dict[str, Any],
+    properties: Optional[NamespaceProperties] = None,
     catalog: Optional[str] = None,
     *args,
     **kwargs,
 ) -> Namespace:
-    """Creates a table namespace with the given name and permissions. Returns
+    """Creates a table namespace with the given name and properties. Returns
     the created namespace. Raises an error if the namespace already exists."""
-    return _get_catalog(catalog).impl.create_namespace(
-        namespace, permissions, *args, **kwargs
+    catalog_obj = get_catalog(catalog)
+    return catalog_obj.impl.create_namespace(
+        namespace,
+        *args,
+        properties=properties,
+        inner=catalog_obj.inner,
+        **kwargs,
     )
 
 
 def alter_namespace(
     namespace: str,
+    *args,
     catalog: Optional[str] = None,
-    permissions: Optional[Dict[str, Any]] = None,
+    properties: Optional[NamespaceProperties] = None,
     new_namespace: Optional[str] = None,
-    *args,
     **kwargs,
 ) -> None:
     """Alter table namespace definition."""
-    _get_catalog(catalog).impl.alter_namespace(
-        namespace, permissions, new_namespace, *args, **kwargs
+    catalog_obj = get_catalog(catalog)
+    catalog_obj.impl.alter_namespace(
+        namespace,
+        *args,
+        properties=properties,
+        new_namespace=new_namespace,
+        inner=catalog_obj.inner,
+        **kwargs,
     )
 
 
 def drop_namespace(
-    namespace: str, catalog: Optional[str] = None, purge: bool = False, *args, **kwargs
+    namespace: str, *args, catalog: Optional[str] = None, purge: bool = False, **kwargs
 ) -> None:
     """Drop the given namespace and all of its tables from the catalog,
     optionally purging them."""
-    _get_catalog(catalog).impl.drop_namespace(namespace, purge, *args, **kwargs)
+    catalog_obj = get_catalog(catalog)
+    catalog_obj.impl.drop_namespace(
+        namespace,
+        *args,
+        purge=purge,
+        inner=catalog_obj.inner,
+        **kwargs,
+    )
 
 
-def default_namespace(catalog: Optional[str] = None) -> str:
+def default_namespace(*args, catalog: Optional[str] = None, **kwargs) -> str:
     """Returns the default namespace for the catalog."""
-    return _get_catalog(catalog).impl.default_namespace()
+    catalog_obj = get_catalog(catalog)
+    return catalog_obj.impl.default_namespace(*args, inner=catalog_obj.inner, **kwargs)

deltacat/catalog/iceberg/__init__.py

@@ -0,0 +1,4 @@
+from deltacat.catalog.iceberg.iceberg_catalog_config import IcebergCatalogConfig
+import deltacat.catalog.iceberg.impl as IcebergCatalog
+
+__all__ = ["IcebergCatalogConfig", "IcebergCatalog"]

deltacat/catalog/iceberg/iceberg_catalog_config.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+from typing import Any, Dict
+
+from attr import dataclass
+from pyiceberg.catalog import CatalogType
+
+
+@dataclass
+class IcebergCatalogConfig:
+    """
+    Configuration properties for Iceberg catalog implementation.
+
+    This class holds the PyIceberg Catalog instance needed for interaction with
+    Iceberg tables and metadata.
+
+    This configuration is passed through to PyIceberg by invoking load_catalog.
+    The Properties provided must match properties accepted by PyIceberg for each catalog type
+    See: :func:`deltacat.catalog.iceberg.initialize`
+
+    Attributes:
+        type: The PyIceberg Catalog instance
+        properties: Dict of properties passed to pyiceberg load_catalog
+    """
+
+    type: CatalogType
+    properties: Dict[str, Any]