deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/catalog/model/catalog.py

@@ -9,11 +9,8 @@ from functools import partial
 import ray
 
 from deltacat import logs
-from deltacat.annotations import ExperimentalAPI
-from deltacat.catalog.main import impl as DeltaCatCatalog
-from deltacat.catalog.iceberg import impl as IcebergCatalog
-from deltacat.catalog import CatalogProperties
-from deltacat.catalog.iceberg import IcebergCatalogConfig
+from deltacat.catalog.main import impl as dcat
+from deltacat.catalog.model.properties import CatalogProperties
 from deltacat.constants import DEFAULT_CATALOG
 
 all_catalogs: Optional[ray.actor.ActorHandle] = None
@@ -22,14 +19,20 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 class Catalog:
-    def __init__(self, impl: ModuleType = DeltaCatCatalog, *args, **kwargs):
+    def __init__(
+        self,
+        config: Optional[Union[CatalogProperties, Any]] = None,
+        impl: ModuleType = dcat,
+        *args,
+        **kwargs,
+    ):
         """
         Constructor for a Catalog.
 
-        Invokes `impl.initialize(*args, **kwargs)` and stores its return value
-        in the `inner` property, which captures all state required to
-        deterministically reconstruct this Catalog instance on any node (and
-        must therefore be pickleable by Ray cloudpickle).
+        Invokes `impl.initialize(config, *args, **kwargs)` and stores its
+        return value in the `inner` property. This captures all state required
+        to deterministically reconstruct this Catalog instance on any node, and
+        must be pickleable by Ray cloudpickle.
         """
         if not isinstance(self, Catalog):
             # self may contain the tuple returned from __reduce__ (ray pickle bug?)
@@ -40,32 +43,15 @@ class Catalog:
                 err_msg = f"Expected `self` to be {Catalog}, but found: {self}"
                 raise RuntimeError(err_msg)
 
+        self._config = config
         self._impl = impl
-        self._inner = self._impl.initialize(*args, **kwargs)
+        self._inner = self._impl.initialize(config=config, *args, **kwargs)
         self._args = args
         self._kwargs = kwargs
 
-    @classmethod
-    @ExperimentalAPI
-    def iceberg(cls, config: IcebergCatalogConfig, *args, **kwargs):
-        """
-        !!! ICEBERG SUPPORT IS EXPERIMENTAL !!!
-
-        Factory method to construct a catalog from Iceberg catalog params
-
-        This method is just a wrapper around __init__ with stronger typing. You may still call __init__,
-        plumbing __params__ through as kwargs
-        """
-        return cls(impl=IcebergCatalog, *args, **{"config": config, **kwargs})
-
-    @classmethod
-    def default(cls, config: CatalogProperties, *args, **kwargs):
-        """
-        Factory method to construct a catalog with the default implementation
-
-        Uses CatalogProperties as configuration
-        """
-        return cls(impl=DeltaCatCatalog, *args, **{"config": config, **kwargs})
+    @property
+    def config(self):
+        return self._config
 
     @property
     def impl(self):
@@ -79,7 +65,11 @@ class Catalog:
     def __reduce__(self):
         # instantiated catalogs may fail to pickle, so exclude _inner
         # (e.g. Iceberg catalog w/ unserializable SSLContext from boto3 client)
-        return partial(self.__class__, **self._kwargs), (self._impl, *self._args)
+        return partial(self.__class__, **self._kwargs), (
+            self._config,
+            self._impl,
+            *self._args,
+        )
 
     def __str__(self):
         string_rep = f"{self.__class__.__name__}("
@@ -102,38 +92,62 @@ class Catalogs:
         catalogs: Union[Catalog, Dict[str, Catalog]],
         default: Optional[str] = None,
     ):
+        self._catalogs = {}
+        self._default_catalog_name = None
+        self._default_catalog = None
+        self.update(catalogs, default)
+
+    def all(self) -> Dict[str, Catalog]:
+        return self._catalogs
+
+    def update(
+        self,
+        catalogs: Union[Catalog, Dict[str, Catalog]],
+        default: Optional[str] = None,
+    ) -> None:
         if isinstance(catalogs, Catalog):
             catalogs = {DEFAULT_CATALOG: catalogs}
         elif not isinstance(catalogs, dict):
             raise ValueError(f"Expected Catalog or dict, but found: {catalogs}")
-        self.catalogs: Dict[str, Catalog] = catalogs
+        self._catalogs.update(catalogs)
         if default:
             if default not in catalogs:
                 raise ValueError(
                     f"Default catalog `{default}` not found in: {catalogs}"
                 )
-            self.default_catalog = self.catalogs[default]
+            self._default_catalog = self._catalogs[default]
+            self._default_catalog_name = default
         elif len(catalogs) == 1:
-            self.default_catalog = list(self.catalogs.values())[0]
+            self._default_catalog = list(self._catalogs.values())[0]
         else:
-            self.default_catalog = None
-
-    def all(self) -> Dict[str, Catalog]:
-        return self.catalogs
+            self._default_catalog = None
 
     def names(self) -> List[str]:
-        return list(self.catalogs.keys())
+        return list(self._catalogs.keys())
 
     def put(self, name: str, catalog: Catalog, set_default: bool = False) -> None:
-        self.catalogs[name] = catalog
-        if set_default:
-            self.default_catalog = catalog
+        self._catalogs[name] = catalog
+        if set_default or len(self._catalogs) == 1:
+            self._default_catalog = catalog
 
     def get(self, name) -> Optional[Catalog]:
-        return self.catalogs.get(name)
+        return self._catalogs.get(name)
+
+    def pop(self, name) -> Optional[Catalog]:
+        catalog = self._catalogs.pop(name, None)
+        if catalog and self._default_catalog_name == name:
+            if len(self._catalogs) == 1:
+                self._default_catalog = list(self._catalogs.values())[0]
+            else:
+                self._default_catalog = None
+        return catalog
+
+    def clear(self) -> None:
+        self._catalogs.clear()
+        self._default_catalog = None
 
     def default(self) -> Optional[Catalog]:
-        return self.default_catalog
+        return self._default_catalog
 
 
 def is_initialized(*args, **kwargs) -> bool:
@@ -142,12 +156,9 @@ def is_initialized(*args, **kwargs) -> bool:
     """
     global all_catalogs
 
-    # If ray is not initialized, then Catalogs cannot be initialized
     if not ray.is_initialized():
-        # Any existing actor reference stored in catalog_module must be stale - reset it
+        # Any existing Catalogs actor reference must be stale - reset it
         all_catalogs = None
-        return False
-
     return all_catalogs is not None
 
 
@@ -168,10 +179,10 @@ def raise_if_not_initialized(
 def init(
     catalogs: Union[Dict[str, Catalog], Catalog] = {},
     default: Optional[str] = None,
-    ray_init_args: Dict[str, Any] = None,
+    ray_init_args: Dict[str, Any] = {},
     *,
-    force_reinitialize=False,
-) -> None:
+    force=False,
+) -> Optional[ray.runtime.BaseContext]:
     """
     Initialize DeltaCAT catalogs.
 
@@ -180,18 +191,20 @@ def init(
     :param default: The name of the default Catalog. If only one Catalog is
         provided, it will always be the default.
     :param ray_init_args: Keyword arguments to pass to `ray.init()`.
-    :param force_reinitialize: Whether to force Ray reinitialization.
+    :param force: Whether to force DeltaCAT reinitialization. If True, reruns
+        ray.init(**ray_init_args) and overwrites all previously registered
+        catalogs.
+    :returns: The Ray context object if Ray was initialized, otherwise None.
     """
     global all_catalogs
 
-    if is_initialized() and not force_reinitialize:
+    if is_initialized() and not force:
         logger.warning("DeltaCAT already initialized.")
-        return
-    else:
-        if ray_init_args:
-            ray.init(**ray_init_args)
-        else:
-            ray.init()
+        return None
+
+    # initialize ray (and ignore reinitialization errors)
+    ray_init_args["ignore_reinit_error"] = True
+    context = ray.init(**ray_init_args)
 
     # register custom serializer for catalogs since these may contain
     # unserializable objects like boto3 clients with SSLContext
@@ -201,9 +214,42 @@ def init(
     # TODO(pdames): If no catalogs are provided then re-initialize DeltaCAT
     #  with all catalogs from the last session
     all_catalogs = Catalogs.remote(catalogs=catalogs, default=default)
+    return context
 
 
-def get_catalog(name: Optional[str] = None, **kwargs) -> Catalog:
+def init_local(
+    path: Optional[str] = None,
+    ray_init_args: Dict[str, Any] = {},
+    *,
+    force=False,
+) -> Optional[ray.runtime.BaseContext]:
+    """
+    Initialize DeltaCAT with a default local catalog.
+
+    This is a convenience function that creates a default catalog for local usage.
+    Equivalent to calling init(catalogs={"default": Catalog()}).
+
+    :param path: Optional path for catalog root directory. If not provided, uses
+        the default behavior of CatalogProperties (DELTACAT_ROOT env var or
+        "./.deltacat/").
+    :param ray_init_args: Keyword arguments to pass to `ray.init()`.
+    :param force: Whether to force DeltaCAT reinitialization. If True, reruns
+        ray.init(**ray_init_args) and overwrites all previously registered
+        catalogs.
+    :returns: The Ray context object if Ray was initialized, otherwise None.
+    """
+    from deltacat.catalog.model.properties import CatalogProperties
+
+    config = CatalogProperties(root=path) if path is not None else None
+    return init(
+        catalogs={"default": Catalog(config=config)},
+        default="default",
+        ray_init_args=ray_init_args,
+        force=force,
+    )
+
+
+def get_catalog(name: Optional[str] = None) -> Catalog:
     """
     Get a catalog by name, or the default catalog if no name is provided.
 
@@ -232,7 +278,7 @@ def get_catalog(name: Optional[str] = None, **kwargs) -> Catalog:
     else:
         catalog = ray.get(all_catalogs.default.remote())
         if not catalog:
-            available_catalogs = ray.get(all_catalogs.all.remote()).values()
+            available_catalogs = list(ray.get(all_catalogs.all.remote()).keys())
             raise ValueError(
                 f"Call to get_catalog without name set failed because there "
                 f"is no default Catalog set. Available catalogs: "
@@ -241,17 +287,44 @@ def get_catalog(name: Optional[str] = None, **kwargs) -> Catalog:
     return catalog
 
 
+def clear_catalogs() -> None:
+    """
+    Clear all catalogs from the global map of named catalogs.
+    """
+    if all_catalogs:
+        ray.get(all_catalogs.clear.remote())
+
+
+def pop_catalog(name: str) -> Optional[Catalog]:
+    """
+    Remove a named catalog from the global map of named catalogs.
+
+    Args:
+        name: Name of the catalog to remove.
+
+    Returns:
+        The removed catalog, or None if not found.
+    """
+    global all_catalogs
+
+    if not all_catalogs:
+        return None
+    catalog = ray.get(all_catalogs.pop.remote(name))
+    return catalog
+
+
 def put_catalog(
     name: str,
     catalog: Catalog = None,
     *,
     default: bool = False,
-    ray_init_args: Dict[str, Any] = None,
+    ray_init_args: Dict[str, Any] = {},
     fail_if_exists: bool = False,
     **kwargs,
 ) -> Catalog:
     """
-    Add a named catalog to the global map of named catalogs. Initializes ray if not already initialized.
+    Add a named catalog to the global map of named catalogs. Initializes
+    DeltaCAT if not already initialized.
 
     Args:
         name: Name of the catalog.
@@ -261,8 +334,8 @@ def put_catalog(
         default:  Make this the default catalog if multiple catalogs are
             available. If only one catalog is available, it will always be the
             default.
-        ray_init_args: Ray initialization args (used only if ray not already
-            initialized)
+        ray_init_args: Ray initialization args (used only if ray is not already
+            initialized).
         fail_if_exists: if True, raises an error if a catalog with the given
             name already exists. If False, inserts or replaces the given
             catalog name.
@@ -276,6 +349,8 @@ def put_catalog(
 
     if not catalog:
         catalog = Catalog(**kwargs)
+    if name is None:
+        raise ValueError("Catalog name cannot be None")
 
     # Initialize, if necessary
     if not is_initialized():
@@ -283,25 +358,27 @@ def put_catalog(
         if not default:
             logger.info(
                 f"Calling put_catalog with set_as_default=False, "
-                f"but still setting Catalog {catalog} as default since it is the only catalog."
+                f"but still setting Catalog {catalog} as default since it is "
+                f"the only catalog."
             )
         init({name: catalog}, ray_init_args=ray_init_args)
-        return
+        return catalog
 
     # Fail if fail_if_exists and catalog already exists
     if fail_if_exists:
-        catalog_already_exists = False
         try:
             get_catalog(name)
-            # Note - need to set state catalog_already_exists and throw ValueError later, or else it will be
-            # caught in the except block which is meant to catch the ValueError from get_catalog
-            catalog_already_exists = True
-        except ValueError:
-            pass
-        if catalog_already_exists:
+            # If we get here, catalog exists - raise error
             raise ValueError(
-                f"Failed to put catalog {name} because it already exists and fail_if_exists={fail_if_exists}"
+                f"Failed to put catalog {name} because it already exists and "
+                f"fail_if_exists={fail_if_exists}"
             )
+        except ValueError as e:
+            if "not found" not in str(e):
+                # Re-raise if it's not a "catalog not found" error
+                raise
+            # If catalog doesn't exist, continue normally
+            pass
 
     # Add the catalog (which may overwrite existing if fail_if_exists=False)
     ray.get(all_catalogs.put.remote(name, catalog, default))

deltacat/catalog/model/properties.py

@@ -1,5 +1,9 @@
 from __future__ import annotations
+
 from typing import Optional, Any
+import urllib.parse
+
+import os
 
 import pyarrow
 from deltacat.constants import DELTACAT_ROOT
@@ -8,18 +12,17 @@ from deltacat.utils.filesystem import resolve_path_and_filesystem
 
 
 def get_catalog_properties(
-    *args,
+    *,
     catalog: Optional[CatalogProperties] = None,
     inner: Optional[CatalogProperties] = None,
     **kwargs,
 ) -> CatalogProperties:
     """
-    Helper function to fetch CatalogProperties instance. You are meant to call this by providing your functions
-    kwargs, OR to directly pass through CatalogProperty configuration keys like "root" in kwargs.
+    Helper function to fetch CatalogProperties instance.
 
-    This will look for a CatalogProperty value in the kwargs "catalog" or "inner". If these are found, it returns
-    the CatalogProperty value under that kwarg. Otherwise, it will pass through kwargs to the CatalogProperties
-    constructor.
+    This will look first look for CatalogProperties in either "catalog"
+    or "inner" and otherwise passes all keyword arguments to the
+    CatalogProperties constructor.
     """
     properties = catalog if catalog is not None else inner
     if properties is not None and isinstance(properties, CatalogProperties):
@@ -39,21 +42,22 @@ class CatalogProperties:
     DeltaCAT catalog instance. Properties are set from system environment
     variables unless explicit overrides are provided during initialization.
 
-    Catalog and storage APIs rely on the property catalog to retrieve durable state about the catalog they're
-    working against.
+    Catalog and storage APIs rely on the property catalog to retrieve durable
+    state about the catalog they're working against.
 
     Attributes:
-        root (str): URI string The root path where catalog metadata and data
-            files are stored. Root is determined (in prededence order) by:
-            1. check "root" input argument
-            2. check env variable "DELTACAT_ROOT"
-            3. default to ${cwd}/.deltacat
+        root: The root path for catalog metadata and data storage. Resolved by
+            searching for the root path in the following order:
+            1. "root" constructor input argument
+            2. "DELTACAT_ROOT" system environment variable
+            3. default to "./.deltacat/"
 
         filesystem: The filesystem implementation that should be used for
             reading/writing files. If None, a filesystem will be inferred from
             the catalog root path.
 
-        storage: Storage class implementation (overrides default filesystem storage impl)
+        storage: Storage class implementation (overrides default filesystem
+            storage impl)
     """
 
     def __init__(
@@ -66,21 +70,26 @@ class CatalogProperties:
         Initialize a CatalogProperties instance.
 
         Args:
-            root: A single directory path that serves as the catalog root dir.
+            root: Catalog root directory path. Uses the "DELTACAT_ROOT"
+                system environment variable if not set, and defaults to
+                "./.deltacat/" if this environment variable is not set.
             filesystem: The filesystem implementation that should be used for
                 reading these files. If None, a filesystem will be inferred.
-                If not None, the provided filesystem will still be validated
-                against the provided path to ensure compatibility.
+                If provided, this will be validated for compatibility with the
+                catalog root path.
+            storage: DeltaCAT storage implementation override.
         """
         # set root, using precedence rules described in pydoc
         if root is None:
             # Check environment variables
-            # This is set or defaulted in constants.py
             root = DELTACAT_ROOT
-            if root is None:
-                raise ValueError(
-                    "Expected environment variable DELTACAT_ROOT to be set or defaulted"
-                )
+            if not root:
+                # Default to "./.deltacat/"
+                root = os.path.join(os.getcwd(), ".deltacat")
+
+        # Store the original root with its scheme for reconstruction later
+        self._original_root = root
+        self._original_scheme = urllib.parse.urlparse(root).scheme
 
         resolved_root, resolved_filesystem = resolve_path_and_filesystem(
             path=root,
@@ -105,6 +114,38 @@ class CatalogProperties:
         """
         return self._storage
 
+    def reconstruct_full_path(self, path: str) -> str:
+        """
+        Reconstruct a full path with the original scheme for external readers.
+
+        This addresses GitHub issue #567 by ensuring that cloud storage URIs
+        include the relevant scheme prefix (e.g., s3://) that some file readers
+        require regardless of the filesystem being used to read the file
+        (e.g., Daft).
+
+        Args:
+            path: A path relative to the catalog root or absolute path
+
+        Returns:
+            Full path with appropriate scheme prefix for external readers
+        """
+        # If the path already has a scheme, return it as-is
+        if urllib.parse.urlparse(path).scheme:
+            return path
+
+        # If we don't have an original scheme (local filesystem), return as-is
+        if not self._original_scheme:
+            return path
+
+        # Reconstruct the full path with the original scheme
+        # Handle both absolute and relative paths
+        if path.startswith("/"):
+            # Absolute path - this shouldn't happen normally but handle it
+            return f"{self._original_scheme}:/{path}"
+        else:
+            # Relative path - prepend the s3:// scheme
+            return f"{self._original_scheme}://{path}"
+
     def __str__(self):
         return (
             f"{self.__class__.__name__}(root={self.root}, filesystem={self.filesystem})"