deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/catalog/model/catalog.py

@@ -9,11 +9,8 @@ from functools import partial
 import ray
 
 from deltacat import logs
-from deltacat.annotations import ExperimentalAPI
-from deltacat.catalog.main import impl as DeltacatCatalog
-from deltacat.catalog.iceberg import impl as IcebergCatalog
-from deltacat.catalog import CatalogProperties
-from deltacat.catalog.iceberg import IcebergCatalogConfig
+from deltacat.catalog.main import impl as dcat
+from deltacat.catalog.model.properties import CatalogProperties
 from deltacat.constants import DEFAULT_CATALOG
 
 all_catalogs: Optional[ray.actor.ActorHandle] = None
@@ -22,17 +19,20 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 class Catalog:
-    def __init__(self, impl: ModuleType = DeltacatCatalog, *args, **kwargs):
+    def __init__(
+        self,
+        config: Optional[Union[CatalogProperties, Any]] = None,
+        impl: ModuleType = dcat,
+        *args,
+        **kwargs,
+    ):
         """
         Constructor for a Catalog.
 
-        The args and kwargs here will be plumbed through to the catalog initialize function, and the results
-        are stored in Catalog.inner. Any state which is required (like: metastore root URI, pyiceberg native catalog)
-        MUST be returned by initialize.
-
-        Note: all initialization configuration MUST be pickle-able. When `Catalog` is pickled, _inner is excluded.
-        Instead, we only pass impl/args/kwargs, which are pickled and then _inner is re-constituted by calling __init__.
-        See `ray.util.register_serializer` in Catalogs class.
+        Invokes `impl.initialize(config, *args, **kwargs)` and stores its
+        return value in the `inner` property. This captures all state required
+        to deterministically reconstruct this Catalog instance on any node, and
+        must be pickleable by Ray cloudpickle.
         """
         if not isinstance(self, Catalog):
             # self may contain the tuple returned from __reduce__ (ray pickle bug?)
@@ -43,32 +43,15 @@ class Catalog:
                 err_msg = f"Expected `self` to be {Catalog}, but found: {self}"
                 raise RuntimeError(err_msg)
 
+        self._config = config
         self._impl = impl
-        self._inner = self._impl.initialize(*args, **kwargs)
+        self._inner = self._impl.initialize(config=config, *args, **kwargs)
         self._args = args
         self._kwargs = kwargs
 
-    @classmethod
-    @ExperimentalAPI
-    def iceberg(cls, config: IcebergCatalogConfig, *args, **kwargs):
-        """
-        !!! ICEBERG SUPPORT IS EXPERIMENTAL !!!
-
-        Factory method to construct a catalog from Iceberg catalog params
-
-        This method is just a wrapper around __init__ with stronger typing. You may still call __init__,
-        plumbing __params__ through as kwargs
-        """
-        return cls(impl=IcebergCatalog, *args, **{"config": config, **kwargs})
-
-    @classmethod
-    def default(cls, config: CatalogProperties, *args, **kwargs):
-        """
-        Factory method to construct a catalog with the default implementation
-
-        Uses CatalogProperties as configuration
-        """
-        return cls(impl=DeltacatCatalog, *args, **{"config": config, **kwargs})
+    @property
+    def config(self):
+        return self._config
 
     @property
     def impl(self):
@@ -82,7 +65,11 @@ class Catalog:
     def __reduce__(self):
         # instantiated catalogs may fail to pickle, so exclude _inner
         # (e.g. Iceberg catalog w/ unserializable SSLContext from boto3 client)
-        return partial(self.__class__, **self._kwargs), (self._impl, *self._args)
+        return partial(self.__class__, **self._kwargs), (
+            self._config,
+            self._impl,
+            *self._args,
+        )
 
     def __str__(self):
         string_rep = f"{self.__class__.__name__}("
@@ -104,101 +91,165 @@ class Catalogs:
         self,
         catalogs: Union[Catalog, Dict[str, Catalog]],
         default: Optional[str] = None,
-        *args,
-        **kwargs,
     ):
-        if default and default not in catalogs:
-            raise ValueError(
-                f"Catalog {default} not found " f"in catalogs to register: {catalogs}"
-            )
-        if not catalogs:
-            raise ValueError(
-                f"No catalogs given to register. "
-                f"Please specify one or more catalogs."
-            )
+        self._catalogs = {}
+        self._default_catalog_name = None
+        self._default_catalog = None
+        self.update(catalogs, default)
 
-        # if user only provides single Catalog, override it to be a map with default key
+    def all(self) -> Dict[str, Catalog]:
+        return self._catalogs
+
+    def update(
+        self,
+        catalogs: Union[Catalog, Dict[str, Catalog]],
+        default: Optional[str] = None,
+    ) -> None:
         if isinstance(catalogs, Catalog):
             catalogs = {DEFAULT_CATALOG: catalogs}
-
-        self.catalogs: Dict[str, Catalog] = catalogs
+        elif not isinstance(catalogs, dict):
+            raise ValueError(f"Expected Catalog or dict, but found: {catalogs}")
+        self._catalogs.update(catalogs)
         if default:
-            self.default_catalog = self.catalogs[default]
+            if default not in catalogs:
+                raise ValueError(
+                    f"Default catalog `{default}` not found in: {catalogs}"
+                )
+            self._default_catalog = self._catalogs[default]
+            self._default_catalog_name = default
         elif len(catalogs) == 1:
-            self.default_catalog = list(self.catalogs.values())[0]
+            self._default_catalog = list(self._catalogs.values())[0]
         else:
-            self.default_catalog = None
-
-    def all(self) -> Dict[str, Catalog]:
-        return self.catalogs
+            self._default_catalog = None
 
     def names(self) -> List[str]:
-        return list(self.catalogs.keys())
+        return list(self._catalogs.keys())
 
     def put(self, name: str, catalog: Catalog, set_default: bool = False) -> None:
-        self.catalogs[name] = catalog
-        if set_default:
-            self.default_catalog = catalog
+        self._catalogs[name] = catalog
+        if set_default or len(self._catalogs) == 1:
+            self._default_catalog = catalog
+
+    def get(self, name) -> Optional[Catalog]:
+        return self._catalogs.get(name)
+
+    def pop(self, name) -> Optional[Catalog]:
+        catalog = self._catalogs.pop(name, None)
+        if catalog and self._default_catalog_name == name:
+            if len(self._catalogs) == 1:
+                self._default_catalog = list(self._catalogs.values())[0]
+            else:
+                self._default_catalog = None
+        return catalog
 
-    def get(self, name) -> Catalog:
-        return self.catalogs.get(name)
+    def clear(self) -> None:
+        self._catalogs.clear()
+        self._default_catalog = None
 
     def default(self) -> Optional[Catalog]:
-        return self.default_catalog
+        return self._default_catalog
 
 
 def is_initialized(*args, **kwargs) -> bool:
     """
-    Check if DeltaCAT is initialized
+    Check if DeltaCAT is initialized.
     """
     global all_catalogs
 
-    # If ray is not initialized, then Catalogs cannot be initialized
     if not ray.is_initialized():
-        # Any existing actor reference stored in catalog_module must be stale - reset it
+        # Any existing Catalogs actor reference must be stale - reset it
         all_catalogs = None
-        return False
-
     return all_catalogs is not None
 
 
+def raise_if_not_initialized(
+    err_msg: str = "DeltaCAT is not initialized. Please call `deltacat.init()` and try again.",
+) -> None:
+    """
+    Raises a RuntimeError with the given error message if DeltaCAT is not
+    initialized.
+
+    :param err_msg: Custom error message to raise if DeltaCAT is not
+    initialized. If unspecified, the default error message is used.
+    """
+    if not is_initialized():
+        raise RuntimeError(err_msg)
+
+
 def init(
-    catalogs: Union[Dict[str, Catalog], Catalog],
+    catalogs: Union[Dict[str, Catalog], Catalog] = {},
     default: Optional[str] = None,
-    ray_init_args: Dict[str, Any] = None,
-    *args,
-    force_reinitialize=False,
-    **kwargs,
-) -> None:
+    ray_init_args: Dict[str, Any] = {},
+    *,
+    force=False,
+) -> Optional[ray.runtime.BaseContext]:
     """
     Initialize DeltaCAT catalogs.
 
-    :param catalogs: Either a single Catalog instance or a map of string to Catalog instance
-    :param default: The Catalog to use by default. If only one Catalog is provided, it will
-        be set as the default
-    :param ray_init_args: kwargs to pass to ray initialization
-    :param force_reinitialize: if True, force the reinitialization of Ray. If false, will do nothing if ray already initialized
+    :param catalogs: A single Catalog instance or a map of catalog names to
+        Catalog instances.
+    :param default: The name of the default Catalog. If only one Catalog is
+        provided, it will always be the default.
+    :param ray_init_args: Keyword arguments to pass to `ray.init()`.
+    :param force: Whether to force DeltaCAT reinitialization. If True, reruns
+        ray.init(**ray_init_args) and overwrites all previously registered
+        catalogs.
+    :returns: The Ray context object if Ray was initialized, otherwise None.
     """
     global all_catalogs
 
-    if is_initialized() and not force_reinitialize:
+    if is_initialized() and not force:
         logger.warning("DeltaCAT already initialized.")
-        return
-    else:
-        if ray_init_args:
-            ray.init(**ray_init_args)
-        else:
-            ray.init()
+        return None
+
+    # initialize ray (and ignore reinitialization errors)
+    ray_init_args["ignore_reinit_error"] = True
+    context = ray.init(**ray_init_args)
 
     # register custom serializer for catalogs since these may contain
     # unserializable objects like boto3 clients with SSLContext
     ray.util.register_serializer(
         Catalog, serializer=Catalog.__reduce__, deserializer=Catalog.__init__
     )
+    # TODO(pdames): If no catalogs are provided then re-initialize DeltaCAT
+    #  with all catalogs from the last session
     all_catalogs = Catalogs.remote(catalogs=catalogs, default=default)
+    return context
+
+
+def init_local(
+    path: Optional[str] = None,
+    ray_init_args: Dict[str, Any] = {},
+    *,
+    force=False,
+) -> Optional[ray.runtime.BaseContext]:
+    """
+    Initialize DeltaCAT with a default local catalog.
+
+    This is a convenience function that creates a default catalog for local usage.
+    Equivalent to calling init(catalogs={"default": Catalog()}).
+
+    :param path: Optional path for catalog root directory. If not provided, uses
+        the default behavior of CatalogProperties (DELTACAT_ROOT env var or
+        "./.deltacat/").
+    :param ray_init_args: Keyword arguments to pass to `ray.init()`.
+    :param force: Whether to force DeltaCAT reinitialization. If True, reruns
+        ray.init(**ray_init_args) and overwrites all previously registered
+        catalogs.
+    :returns: The Ray context object if Ray was initialized, otherwise None.
+    """
+    from deltacat.catalog.model.properties import CatalogProperties
+
+    config = CatalogProperties(root=path) if path is not None else None
+    return init(
+        catalogs={"default": Catalog(config=config)},
+        default="default",
+        ray_init_args=ray_init_args,
+        force=force,
+    )
 
 
-def get_catalog(name: Optional[str] = None, **kwargs) -> Catalog:
+def get_catalog(name: Optional[str] = None) -> Catalog:
     """
     Get a catalog by name, or the default catalog if no name is provided.
 
@@ -216,7 +267,6 @@ def get_catalog(name: Optional[str] = None, **kwargs) -> Catalog:
             "`deltacat.init(catalogs={...})` to register one or more "
             "catalogs then retry."
         )
-
     if name is not None:
         catalog = ray.get(all_catalogs.get.remote(name))
         if not catalog:
@@ -225,17 +275,42 @@ def get_catalog(name: Optional[str] = None, **kwargs) -> Catalog:
                 f"Catalog '{name}' not found. Available catalogs: "
                 f"{available_catalogs}."
             )
-        return catalog
-
     else:
         catalog = ray.get(all_catalogs.default.remote())
         if not catalog:
-            available_catalogs = ray.get(all_catalogs.all.remote()).values()
+            available_catalogs = list(ray.get(all_catalogs.all.remote()).keys())
             raise ValueError(
-                f"Call to get_catalog without name set failed because there is no default Catalog set. Available catalogs: "
+                f"Call to get_catalog without name set failed because there "
+                f"is no default Catalog set. Available catalogs: "
                 f"{available_catalogs}."
             )
-        return catalog
+    return catalog
+
+
+def clear_catalogs() -> None:
+    """
+    Clear all catalogs from the global map of named catalogs.
+    """
+    if all_catalogs:
+        ray.get(all_catalogs.clear.remote())
+
+
+def pop_catalog(name: str) -> Optional[Catalog]:
+    """
+    Remove a named catalog from the global map of named catalogs.
+
+    Args:
+        name: Name of the catalog to remove.
+
+    Returns:
+        The removed catalog, or None if not found.
+    """
+    global all_catalogs
+
+    if not all_catalogs:
+        return None
+    catalog = ray.get(all_catalogs.pop.remote(name))
+    return catalog
 
 
 def put_catalog(
@@ -243,48 +318,68 @@ def put_catalog(
     catalog: Catalog = None,
     *,
     default: bool = False,
-    ray_init_args: Dict[str, Any] = None,
+    ray_init_args: Dict[str, Any] = {},
     fail_if_exists: bool = False,
     **kwargs,
-) -> None:
+) -> Catalog:
     """
-    Add a named catalog to the global map of named catalogs. Initializes ray if not already initialized.
+    Add a named catalog to the global map of named catalogs. Initializes
+    DeltaCAT if not already initialized.
 
     Args:
-        name: name of catalog
-        catalog: catalog instance to use, if provided
-        default:  Make this the default catalog if multiple catalogs are available.
-            ignored if this is the only catalog available, since it will always be the default catalog.
-        ray_init_args: ray initialization args (used only if ray not already initialized)
-        fail_if_exists: if True, raises KeyError if the catalog name already exists. Otherwise, overwrite catalog
+        name: Name of the catalog.
+        catalog: Catalog instance to use. If none is provided, then all
+            additional keyword arguments will be forwarded to
+            `CatalogProperties` for a default DeltaCAT native Catalog.
+        default:  Make this the default catalog if multiple catalogs are
+            available. If only one catalog is available, it will always be the
+            default.
+        ray_init_args: Ray initialization args (used only if ray is not already
+            initialized).
+        fail_if_exists: if True, raises an error if a catalog with the given
+            name already exists. If False, inserts or replaces the given
+            catalog name.
+        kwargs: Additional keyword arguments to forward to `CatalogProperties`
+            for a default DeltaCAT native Catalog.
+
+    Returns:
+        The catalog put in the named catalog map.
     """
     global all_catalogs
 
+    if not catalog:
+        catalog = Catalog(**kwargs)
+    if name is None:
+        raise ValueError("Catalog name cannot be None")
+
     # Initialize, if necessary
     if not is_initialized():
-        # NOTE - since we are initializing with a single catalog, it will be set to the default
+        # We are initializing a single catalog - make it the default
         if not default:
             logger.info(
                 f"Calling put_catalog with set_as_default=False, "
-                f"but still setting Catalog {catalog} as default since it is the only catalog."
+                f"but still setting Catalog {catalog} as default since it is "
+                f"the only catalog."
             )
         init({name: catalog}, ray_init_args=ray_init_args)
-        return
+        return catalog
 
     # Fail if fail_if_exists and catalog already exists
     if fail_if_exists:
-        catalog_already_exists = False
         try:
             get_catalog(name)
-            # Note - need to set state catalog_already_exists and throw ValueError later, or else it will be
-            # caught in the except block which is meant to catch the ValueError from get_catalog
-            catalog_already_exists = True
-        except ValueError:
-            pass
-        if catalog_already_exists:
+            # If we get here, catalog exists - raise error
             raise ValueError(
-                f"Failed to put catalog {name} because it already exists and fail_if_exists={fail_if_exists}"
+                f"Failed to put catalog {name} because it already exists and "
+                f"fail_if_exists={fail_if_exists}"
             )
+        except ValueError as e:
+            if "not found" not in str(e):
+                # Re-raise if it's not a "catalog not found" error
+                raise
+            # If catalog doesn't exist, continue normally
+            pass
 
     # Add the catalog (which may overwrite existing if fail_if_exists=False)
     ray.get(all_catalogs.put.remote(name, catalog, default))
+    return catalog

deltacat/catalog/model/properties.py

@@ -1,5 +1,9 @@
 from __future__ import annotations
+
 from typing import Optional, Any
+import urllib.parse
+
+import os
 
 import pyarrow
 from deltacat.constants import DELTACAT_ROOT
@@ -8,18 +12,17 @@ from deltacat.utils.filesystem import resolve_path_and_filesystem
 
 
 def get_catalog_properties(
-    *args,
+    *,
     catalog: Optional[CatalogProperties] = None,
     inner: Optional[CatalogProperties] = None,
     **kwargs,
 ) -> CatalogProperties:
     """
-    Helper function to fetch CatalogProperties instance. You are meant to call this by providing your functions
-    kwargs, OR to directly pass through CatalogProperty configuration keys like "root" in kwargs.
+    Helper function to fetch CatalogProperties instance.
 
-    This will look for a CatalogProperty value in the kwargs "catalog" or "inner". If these are found, it returns
-    the CatalogProperty value under that kwarg. Otherwise, it will pass through kwargs to the CatalogProperties
-    constructor.
+    This will look first look for CatalogProperties in either "catalog"
+    or "inner" and otherwise passes all keyword arguments to the
+    CatalogProperties constructor.
     """
     properties = catalog if catalog is not None else inner
     if properties is not None and isinstance(properties, CatalogProperties):
@@ -39,21 +42,22 @@ class CatalogProperties:
     DeltaCAT catalog instance. Properties are set from system environment
     variables unless explicit overrides are provided during initialization.
 
-    Catalog and storage APIs rely on the property catalog to retrieve durable state about the catalog they're
-    working against.
+    Catalog and storage APIs rely on the property catalog to retrieve durable
+    state about the catalog they're working against.
 
     Attributes:
-        root (str): URI string The root path where catalog metadata and data
-            files are stored. Root is determined (in prededence order) by:
-            1. check kwargs for "root"
-            2. check env variable "DELTACAT_ROOT"
-            3. default to ${cwd}/.deltacat
+        root: The root path for catalog metadata and data storage. Resolved by
+            searching for the root path in the following order:
+            1. "root" constructor input argument
+            2. "DELTACAT_ROOT" system environment variable
+            3. default to "./.deltacat/"
 
         filesystem: The filesystem implementation that should be used for
             reading/writing files. If None, a filesystem will be inferred from
             the catalog root path.
 
-        storage: Storage class implementation (overrides default filesystem storage impl)
+        storage: Storage class implementation (overrides default filesystem
+            storage impl)
     """
 
     def __init__(
@@ -61,28 +65,31 @@ class CatalogProperties:
         root: Optional[str] = None,
         filesystem: Optional[pyarrow.fs.FileSystem] = None,
         storage=None,
-        *args,
-        **kwargs,
     ):
         """
         Initialize a CatalogProperties instance.
 
         Args:
-            root: A single directory path that serves as the catalog root dir.
+            root: Catalog root directory path. Uses the "DELTACAT_ROOT"
+                system environment variable if not set, and defaults to
+                "./.deltacat/" if this environment variable is not set.
             filesystem: The filesystem implementation that should be used for
                 reading these files. If None, a filesystem will be inferred.
-                If not None, the provided filesystem will still be validated
-                against the provided path to ensure compatibility.
+                If provided, this will be validated for compatibility with the
+                catalog root path.
+            storage: DeltaCAT storage implementation override.
         """
         # set root, using precedence rules described in pydoc
         if root is None:
             # Check environment variables
-            # This is set or defaulted in constants.py
             root = DELTACAT_ROOT
-            if root is None:
-                raise ValueError(
-                    "Expected environment variable DELTACAT_ROOT to be set or defaulted"
-                )
+            if not root:
+                # Default to "./.deltacat/"
+                root = os.path.join(os.getcwd(), ".deltacat")
+
+        # Store the original root with its scheme for reconstruction later
+        self._original_root = root
+        self._original_scheme = urllib.parse.urlparse(root).scheme
 
         resolved_root, resolved_filesystem = resolve_path_and_filesystem(
             path=root,
@@ -107,6 +114,38 @@ class CatalogProperties:
         """
         return self._storage
 
+    def reconstruct_full_path(self, path: str) -> str:
+        """
+        Reconstruct a full path with the original scheme for external readers.
+
+        This addresses GitHub issue #567 by ensuring that cloud storage URIs
+        include the relevant scheme prefix (e.g., s3://) that some file readers
+        require regardless of the filesystem being used to read the file
+        (e.g., Daft).
+
+        Args:
+            path: A path relative to the catalog root or absolute path
+
+        Returns:
+            Full path with appropriate scheme prefix for external readers
+        """
+        # If the path already has a scheme, return it as-is
+        if urllib.parse.urlparse(path).scheme:
+            return path
+
+        # If we don't have an original scheme (local filesystem), return as-is
+        if not self._original_scheme:
+            return path
+
+        # Reconstruct the full path with the original scheme
+        # Handle both absolute and relative paths
+        if path.startswith("/"):
+            # Absolute path - this shouldn't happen normally but handle it
+            return f"{self._original_scheme}:/{path}"
+        else:
+            # Relative path - prepend the s3:// scheme
+            return f"{self._original_scheme}://{path}"
+
     def __str__(self):
         return (
             f"{self.__class__.__name__}(root={self.root}, filesystem={self.filesystem})"

deltacat/compute/__init__.py

@@ -0,0 +1,14 @@
+from deltacat.compute.jobs.client import (
+    DeltaCatJobClient,
+    job_client,
+    local_job_client,
+)
+
+from ray.job_submission import JobStatus
+
+__all__ = [
+    "job_client",
+    "local_job_client",
+    "DeltaCatJobClient",
+    "JobStatus",
+]