deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b11__py3-none-any.whl

deltacat/__init__.py

@@ -8,7 +8,7 @@ from deltacat.api import (
     list,
     put,
 )
-from deltacat.catalog.delegate import (
+from deltacat.catalog import (  # noqa: F401
     alter_namespace,
     alter_table,
     create_namespace,
@@ -27,17 +27,18 @@ from deltacat.catalog.delegate import (
     table_exists,
     truncate_table,
     write_to_table,
-)
-from deltacat.catalog.model.catalog import (  # noqa: F401
-    Catalog,
-    Catalogs,
-    raise_if_not_initialized,
-    is_initialized,
     init,
+    is_initialized,
+    clear_catalogs,
     get_catalog,
+    get_catalog_properties,
+    pop_catalog,
     put_catalog,
+    raise_if_not_initialized,
+    Catalog,
+    CatalogProperties,
+    TableDefinition,
 )
-from deltacat.catalog.model.table_definition import TableDefinition
 from deltacat.compute import (
     job_client,
     local_job_client,
@@ -60,7 +61,6 @@ from deltacat.storage import (
     SortScheme,
     NullOrder,
 )
-from deltacat.storage.rivulet import Dataset as RivDataset, Datatype as RivDatatype
 from deltacat.types.media import (
     ContentEncoding,
     ContentType,
@@ -73,7 +73,9 @@ from deltacat.utils.url import DeltaCatUrl
 
 __iceberg__ = []
 if importlib.util.find_spec("pyiceberg") is not None:
-    from deltacat.catalog.iceberg import impl as IcebergCatalog  # noqa: F401
+    from deltacat.experimental.catalog.iceberg import (  # noqa: F401
+        impl as IcebergCatalog,
+    )
 
     __iceberg__ = [
         "IcebergCatalog",
@@ -81,7 +83,7 @@ if importlib.util.find_spec("pyiceberg") is not None:
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "2.0.0b10"
+__version__ = "2.0.0b11"
 
 
 __all__ = [
@@ -110,12 +112,16 @@ __all__ = [
     "default_namespace",
     "write_to_table",
     "read_table",
+    "init",
+    "is_initialized",
+    "clear_catalogs",
     "get_catalog",
+    "get_catalog_properties",
+    "pop_catalog",
     "put_catalog",
     "raise_if_not_initialized",
-    "is_initialized",
-    "init",
     "Catalog",
+    "CatalogProperties",
     "ContentType",
     "ContentEncoding",
     "Dataset",
@@ -123,8 +129,6 @@ __all__ = [
     "DatastoreType",
     "DeltaCatUrl",
     "DistributedDataset",
-    "RivDataset",
-    "RivDatatype",
     "Field",
     "LifecycleState",
     "ListResult",

deltacat/benchmarking/benchmark_engine.py

@@ -4,8 +4,10 @@ from contextlib import contextmanager
 from typing import Generator, Tuple
 
 from deltacat.benchmarking.benchmark_report import BenchmarkMetric, BenchmarkStep
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
 
 
 @contextmanager

deltacat/benchmarking/test_benchmark_pipeline.py

@@ -1,10 +1,12 @@
 import math
 from random import shuffle
 import pytest
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet.schema.datatype import Datatype
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 from deltacat.benchmarking.benchmark_engine import BenchmarkEngine
 from deltacat.benchmarking.benchmark_report import BenchmarkRun, BenchmarkReport
 from deltacat.benchmarking.benchmark_suite import BenchmarkSuite

deltacat/catalog/__init__.py

@@ -1,14 +1,71 @@
+from deltacat.catalog.delegate import (
+    alter_namespace,
+    alter_table,
+    create_namespace,
+    create_table,
+    default_namespace,
+    drop_namespace,
+    drop_table,
+    get_namespace,
+    get_table,
+    list_namespaces,
+    list_tables,
+    namespace_exists,
+    read_table,
+    refresh_table,
+    rename_table,
+    table_exists,
+    truncate_table,
+    write_to_table,
+)
+from deltacat.catalog.model.catalog import (  # noqa: F401
+    all_catalogs,
+    init,
+    is_initialized,
+    clear_catalogs,
+    get_catalog,
+    pop_catalog,
+    put_catalog,
+    raise_if_not_initialized,
+    Catalog,
+)
 from deltacat.catalog.model.properties import (  # noqa: F401
     CatalogProperties,
     get_catalog_properties,
 )
-from deltacat.catalog.model.catalog import Catalog, Catalogs  # noqa: F401
-from deltacat.catalog.main import impl as DeltacatCatalog
+from deltacat.catalog.model.table_definition import TableDefinition
+from deltacat.catalog.main import impl as dcat
 
 __all__ = [
-    "CatalogProperties",
+    "alter_namespace",
+    "alter_table",
+    "create_namespace",
+    "create_table",
+    "default_namespace",
+    "drop_namespace",
+    "drop_table",
+    "get_namespace",
+    "get_table",
+    "list_namespaces",
+    "list_tables",
+    "namespace_exists",
+    "read_table",
+    "refresh_table",
+    "rename_table",
+    "table_exists",
+    "truncate_table",
+    "write_to_table",
+    "all_catalogs",
+    "init",
+    "is_initialized",
+    "clear_catalogs",
+    "get_catalog",
     "get_catalog_properties",
+    "pop_catalog",
+    "put_catalog",
+    "raise_if_not_initialized",
+    "dcat",
     "Catalog",
-    "Catalogs",
-    "DeltacatCatalog",
+    "CatalogProperties",
+    "TableDefinition",
 ]

deltacat/catalog/main/impl.py

@@ -3,7 +3,7 @@ import logging
 
 import deltacat as dc
 
-from deltacat.catalog import CatalogProperties
+from deltacat.catalog.model.properties import CatalogProperties
 from deltacat.exceptions import (
     NamespaceAlreadyExistsError,
     StreamNotFoundError,
@@ -42,20 +42,30 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 """
 Default Catalog interface implementation using DeltaCAT native storage.
 
-When this is used by `delegate.py` the `Catalog` implementation `inner`
-property will be set to the value returned from `intialize`.
+The functions here should not be invoked directly, but should instead be
+invoked through `delegate.py` (e.g., to support passing catalog's by name, and
+to ensure that each initialized `Catalog` implementation has its `inner`
+property set to the `CatalogProperties` returned from `initialize()`).
 
-`CatalogProperties` has all state required to implement catalog functions,
-such as metastore root URI.
+The `CatalogProperties` instance returned by `initialize()` contains all
+durable state required to deterministically reconstruct the associated DeltaCAT
+native `Catalog` implementation (e.g., the root URI for the catalog metastore).
 """
 
 
 # catalog functions
-def initialize(config: CatalogProperties = None, *args, **kwargs) -> CatalogProperties:
+def initialize(
+    config: Optional[CatalogProperties] = None,
+    *args,
+    **kwargs,
+) -> CatalogProperties:
     """
-    Initializes the data catalog with the given arguments.
+    Performs any required one-time initialization and validation of this
+    catalog implementation based on the input configuration. If no config
+    instance is given, a new `CatalogProperties` instance is constructed
+    using the given keyword arguments.
 
-    returns CatalogProperties as the "inner" state value for a DC native catalog
+    Returns the input config if given, and the newly created config otherwise.
     """
     if config is not None:
         if not isinstance(config, CatalogProperties):

deltacat/catalog/model/catalog.py

@@ -9,11 +9,8 @@ from functools import partial
 import ray
 
 from deltacat import logs
-from deltacat.annotations import ExperimentalAPI
-from deltacat.catalog.main import impl as DeltaCatCatalog
-from deltacat.catalog.iceberg import impl as IcebergCatalog
-from deltacat.catalog import CatalogProperties
-from deltacat.catalog.iceberg import IcebergCatalogConfig
+from deltacat.catalog.main import impl as dcat
+from deltacat.catalog.model.properties import CatalogProperties
 from deltacat.constants import DEFAULT_CATALOG
 
 all_catalogs: Optional[ray.actor.ActorHandle] = None
@@ -22,14 +19,20 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 class Catalog:
-    def __init__(self, impl: ModuleType = DeltaCatCatalog, *args, **kwargs):
+    def __init__(
+        self,
+        config: Optional[Union[CatalogProperties, Any]] = None,
+        impl: ModuleType = dcat,
+        *args,
+        **kwargs,
+    ):
         """
         Constructor for a Catalog.
 
-        Invokes `impl.initialize(*args, **kwargs)` and stores its return value
-        in the `inner` property, which captures all state required to
-        deterministically reconstruct this Catalog instance on any node (and
-        must therefore be pickleable by Ray cloudpickle).
+        Invokes `impl.initialize(config, *args, **kwargs)` and stores its
+        return value in the `inner` property. This captures all state required
+        to deterministically reconstruct this Catalog instance on any node, and
+        must be pickleable by Ray cloudpickle.
         """
         if not isinstance(self, Catalog):
             # self may contain the tuple returned from __reduce__ (ray pickle bug?)
@@ -40,32 +43,15 @@ class Catalog:
                 err_msg = f"Expected `self` to be {Catalog}, but found: {self}"
                 raise RuntimeError(err_msg)
 
+        self._config = config
         self._impl = impl
-        self._inner = self._impl.initialize(*args, **kwargs)
+        self._inner = self._impl.initialize(config=config, *args, **kwargs)
         self._args = args
         self._kwargs = kwargs
 
-    @classmethod
-    @ExperimentalAPI
-    def iceberg(cls, config: IcebergCatalogConfig, *args, **kwargs):
-        """
-        !!! ICEBERG SUPPORT IS EXPERIMENTAL !!!
-
-        Factory method to construct a catalog from Iceberg catalog params
-
-        This method is just a wrapper around __init__ with stronger typing. You may still call __init__,
-        plumbing __params__ through as kwargs
-        """
-        return cls(impl=IcebergCatalog, *args, **{"config": config, **kwargs})
-
-    @classmethod
-    def default(cls, config: CatalogProperties, *args, **kwargs):
-        """
-        Factory method to construct a catalog with the default implementation
-
-        Uses CatalogProperties as configuration
-        """
-        return cls(impl=DeltaCatCatalog, *args, **{"config": config, **kwargs})
+    @property
+    def config(self):
+        return self._config
 
     @property
     def impl(self):
@@ -79,7 +65,11 @@ class Catalog:
     def __reduce__(self):
         # instantiated catalogs may fail to pickle, so exclude _inner
         # (e.g. Iceberg catalog w/ unserializable SSLContext from boto3 client)
-        return partial(self.__class__, **self._kwargs), (self._impl, *self._args)
+        return partial(self.__class__, **self._kwargs), (
+            self._config,
+            self._impl,
+            *self._args,
+        )
 
     def __str__(self):
         string_rep = f"{self.__class__.__name__}("
@@ -102,38 +92,62 @@ class Catalogs:
         catalogs: Union[Catalog, Dict[str, Catalog]],
         default: Optional[str] = None,
     ):
+        self._catalogs = {}
+        self._default_catalog_name = None
+        self._default_catalog = None
+        self.update(catalogs, default)
+
+    def all(self) -> Dict[str, Catalog]:
+        return self._catalogs
+
+    def update(
+        self,
+        catalogs: Union[Catalog, Dict[str, Catalog]],
+        default: Optional[str] = None,
+    ) -> None:
         if isinstance(catalogs, Catalog):
             catalogs = {DEFAULT_CATALOG: catalogs}
         elif not isinstance(catalogs, dict):
             raise ValueError(f"Expected Catalog or dict, but found: {catalogs}")
-        self.catalogs: Dict[str, Catalog] = catalogs
+        self._catalogs.update(catalogs)
         if default:
             if default not in catalogs:
                 raise ValueError(
                     f"Default catalog `{default}` not found in: {catalogs}"
                 )
-            self.default_catalog = self.catalogs[default]
+            self._default_catalog = self._catalogs[default]
+            self._default_catalog_name = default
         elif len(catalogs) == 1:
-            self.default_catalog = list(self.catalogs.values())[0]
+            self._default_catalog = list(self._catalogs.values())[0]
         else:
-            self.default_catalog = None
-
-    def all(self) -> Dict[str, Catalog]:
-        return self.catalogs
+            self._default_catalog = None
 
     def names(self) -> List[str]:
-        return list(self.catalogs.keys())
+        return list(self._catalogs.keys())
 
     def put(self, name: str, catalog: Catalog, set_default: bool = False) -> None:
-        self.catalogs[name] = catalog
+        self._catalogs[name] = catalog
         if set_default:
-            self.default_catalog = catalog
+            self._default_catalog = catalog
 
     def get(self, name) -> Optional[Catalog]:
-        return self.catalogs.get(name)
+        return self._catalogs.get(name)
+
+    def pop(self, name) -> Optional[Catalog]:
+        catalog = self._catalogs.pop(name, None)
+        if catalog and self._default_catalog_name == name:
+            if len(self._catalogs) == 1:
+                self._default_catalog = list(self._catalogs.values())[0]
+            else:
+                self._default_catalog = None
+        return catalog
+
+    def clear(self) -> None:
+        self._catalogs.clear()
+        self._default_catalog = None
 
     def default(self) -> Optional[Catalog]:
-        return self.default_catalog
+        return self._default_catalog
 
 
 def is_initialized(*args, **kwargs) -> bool:
@@ -142,12 +156,9 @@ def is_initialized(*args, **kwargs) -> bool:
     """
     global all_catalogs
 
-    # If ray is not initialized, then Catalogs cannot be initialized
     if not ray.is_initialized():
-        # Any existing actor reference stored in catalog_module must be stale - reset it
+        # Any existing Catalogs actor reference must be stale - reset it
         all_catalogs = None
-        return False
-
     return all_catalogs is not None
 
 
@@ -168,9 +179,9 @@ def raise_if_not_initialized(
 def init(
     catalogs: Union[Dict[str, Catalog], Catalog] = {},
     default: Optional[str] = None,
-    ray_init_args: Dict[str, Any] = None,
+    ray_init_args: Dict[str, Any] = {},
     *,
-    force_reinitialize=False,
+    force=False,
 ) -> None:
     """
     Initialize DeltaCAT catalogs.
@@ -180,18 +191,19 @@ def init(
     :param default: The name of the default Catalog. If only one Catalog is
         provided, it will always be the default.
     :param ray_init_args: Keyword arguments to pass to `ray.init()`.
-    :param force_reinitialize: Whether to force Ray reinitialization.
+    :param force: Whether to force DeltaCAT reinitialization. If True, reruns
+        ray.init(**ray_init_args) and overwrites all previously registered
+        catalogs.
     """
     global all_catalogs
 
-    if is_initialized() and not force_reinitialize:
+    if is_initialized() and not force:
         logger.warning("DeltaCAT already initialized.")
         return
-    else:
-        if ray_init_args:
-            ray.init(**ray_init_args)
-        else:
-            ray.init()
+
+    # initialize ray (and ignore reinitialization errors)
+    ray_init_args["ignore_reinit_error"] = True
+    ray.init(**ray_init_args)
 
     # register custom serializer for catalogs since these may contain
     # unserializable objects like boto3 clients with SSLContext
@@ -203,7 +215,7 @@ def init(
     all_catalogs = Catalogs.remote(catalogs=catalogs, default=default)
 
 
-def get_catalog(name: Optional[str] = None, **kwargs) -> Catalog:
+def get_catalog(name: Optional[str] = None) -> Catalog:
     """
     Get a catalog by name, or the default catalog if no name is provided.
 
@@ -241,17 +253,44 @@ def get_catalog(name: Optional[str] = None, **kwargs) -> Catalog:
     return catalog
 
 
+def clear_catalogs() -> None:
+    """
+    Clear all catalogs from the global map of named catalogs.
+    """
+    if all_catalogs:
+        ray.get(all_catalogs.clear.remote())
+
+
+def pop_catalog(name: str) -> Optional[Catalog]:
+    """
+    Remove a named catalog from the global map of named catalogs.
+
+    Args:
+        name: Name of the catalog to remove.
+
+    Returns:
+        The removed catalog, or None if not found.
+    """
+    global all_catalogs
+
+    if not all_catalogs:
+        return None
+    catalog = ray.get(all_catalogs.pop.remote(name))
+    return catalog
+
+
 def put_catalog(
     name: str,
     catalog: Catalog = None,
     *,
     default: bool = False,
-    ray_init_args: Dict[str, Any] = None,
+    ray_init_args: Dict[str, Any] = {},
     fail_if_exists: bool = False,
     **kwargs,
 ) -> Catalog:
     """
-    Add a named catalog to the global map of named catalogs. Initializes ray if not already initialized.
+    Add a named catalog to the global map of named catalogs. Initializes
+    DeltaCAT if not already initialized.
 
     Args:
         name: Name of the catalog.
@@ -261,8 +300,8 @@ def put_catalog(
         default:  Make this the default catalog if multiple catalogs are
             available. If only one catalog is available, it will always be the
             default.
-        ray_init_args: Ray initialization args (used only if ray not already
-            initialized)
+        ray_init_args: Ray initialization args (used only if ray is not already
+            initialized).
         fail_if_exists: if True, raises an error if a catalog with the given
             name already exists. If False, inserts or replaces the given
             catalog name.
@@ -276,6 +315,8 @@ def put_catalog(
 
     if not catalog:
         catalog = Catalog(**kwargs)
+    if name is None:
+        raise ValueError("Catalog name cannot be None")
 
     # Initialize, if necessary
     if not is_initialized():
@@ -283,25 +324,22 @@ def put_catalog(
         if not default:
             logger.info(
                 f"Calling put_catalog with set_as_default=False, "
-                f"but still setting Catalog {catalog} as default since it is the only catalog."
+                f"but still setting Catalog {catalog} as default since it is "
+                f"the only catalog."
             )
         init({name: catalog}, ray_init_args=ray_init_args)
-        return
+        return catalog
 
     # Fail if fail_if_exists and catalog already exists
     if fail_if_exists:
-        catalog_already_exists = False
         try:
             get_catalog(name)
-            # Note - need to set state catalog_already_exists and throw ValueError later, or else it will be
-            # caught in the except block which is meant to catch the ValueError from get_catalog
-            catalog_already_exists = True
         except ValueError:
             pass
-        if catalog_already_exists:
-            raise ValueError(
-                f"Failed to put catalog {name} because it already exists and fail_if_exists={fail_if_exists}"
-            )
+        raise ValueError(
+            f"Failed to put catalog {name} because it already exists and "
+            f"fail_if_exists={fail_if_exists}"
+        )
 
     # Add the catalog (which may overwrite existing if fail_if_exists=False)
     ray.get(all_catalogs.put.remote(name, catalog, default))

deltacat/catalog/model/properties.py

@@ -1,6 +1,9 @@
 from __future__ import annotations
+
 from typing import Optional, Any
 
+import os
+
 import pyarrow
 from deltacat.constants import DELTACAT_ROOT
 
@@ -8,18 +11,17 @@ from deltacat.utils.filesystem import resolve_path_and_filesystem
 
 
 def get_catalog_properties(
-    *args,
+    *,
     catalog: Optional[CatalogProperties] = None,
     inner: Optional[CatalogProperties] = None,
     **kwargs,
 ) -> CatalogProperties:
     """
-    Helper function to fetch CatalogProperties instance. You are meant to call this by providing your functions
-    kwargs, OR to directly pass through CatalogProperty configuration keys like "root" in kwargs.
+    Helper function to fetch CatalogProperties instance.
 
-    This will look for a CatalogProperty value in the kwargs "catalog" or "inner". If these are found, it returns
-    the CatalogProperty value under that kwarg. Otherwise, it will pass through kwargs to the CatalogProperties
-    constructor.
+    This will look first look for CatalogProperties in either "catalog"
+    or "inner" and otherwise passes all keyword arguments to the
+    CatalogProperties constructor.
     """
     properties = catalog if catalog is not None else inner
     if properties is not None and isinstance(properties, CatalogProperties):
@@ -39,21 +41,22 @@ class CatalogProperties:
     DeltaCAT catalog instance. Properties are set from system environment
     variables unless explicit overrides are provided during initialization.
 
-    Catalog and storage APIs rely on the property catalog to retrieve durable state about the catalog they're
-    working against.
+    Catalog and storage APIs rely on the property catalog to retrieve durable
+    state about the catalog they're working against.
 
     Attributes:
-        root (str): URI string The root path where catalog metadata and data
-            files are stored. Root is determined (in prededence order) by:
-            1. check "root" input argument
-            2. check env variable "DELTACAT_ROOT"
-            3. default to ${cwd}/.deltacat
+        root: The root path for catalog metadata and data storage. Resolved by
+            searching for the root path in the following order:
+            1. "root" constructor input argument
+            2. "DELTACAT_ROOT" system environment variable
+            3. default to "./.deltacat/"
 
         filesystem: The filesystem implementation that should be used for
             reading/writing files. If None, a filesystem will be inferred from
             the catalog root path.
 
-        storage: Storage class implementation (overrides default filesystem storage impl)
+        storage: Storage class implementation (overrides default filesystem
+            storage impl)
     """
 
     def __init__(
@@ -66,21 +69,21 @@ class CatalogProperties:
         Initialize a CatalogProperties instance.
 
         Args:
-            root: A single directory path that serves as the catalog root dir.
+            root: Catalog root directory path. Uses the "DELTACAT_ROOT"
+                system environment variable if not set, and defaults to
+                "./.deltacat/" if this environment variable is not set.
             filesystem: The filesystem implementation that should be used for
                 reading these files. If None, a filesystem will be inferred.
-                If not None, the provided filesystem will still be validated
-                against the provided path to ensure compatibility.
+                If provided, this will be validated for compatibility with the
+                catalog root path.
         """
         # set root, using precedence rules described in pydoc
         if root is None:
             # Check environment variables
-            # This is set or defaulted in constants.py
             root = DELTACAT_ROOT
-            if root is None:
-                raise ValueError(
-                    "Expected environment variable DELTACAT_ROOT to be set or defaulted"
-                )
+            if not root:
+                # Default to "./.deltacat/"
+                root = os.path.join(os.getcwd(), ".deltacat")
 
         resolved_root, resolved_filesystem = resolve_path_and_filesystem(
             path=root,