deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/catalog/model/catalog.py

@@ -1,83 +1,385 @@
 # Allow self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from typing import Any, Dict, List, Optional
+import logging
+from types import ModuleType
 
+from typing import Any, Dict, List, Optional, Union
+from functools import partial
 import ray
 
-from deltacat.catalog import interface as catalog_interface
+from deltacat import logs
+from deltacat.catalog.main import impl as dcat
+from deltacat.catalog.model.properties import CatalogProperties
+from deltacat.constants import DEFAULT_CATALOG
 
-all_catalogs: Optional[Catalogs] = None
+all_catalogs: Optional[ray.actor.ActorHandle] = None
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 class Catalog:
-    def __init__(self, impl=catalog_interface, *args, **kwargs):
+    def __init__(
+        self,
+        config: Optional[Union[CatalogProperties, Any]] = None,
+        impl: ModuleType = dcat,
+        *args,
+        **kwargs,
+    ):
+        """
+        Constructor for a Catalog.
+
+        Invokes `impl.initialize(config, *args, **kwargs)` and stores its
+        return value in the `inner` property. This captures all state required
+        to deterministically reconstruct this Catalog instance on any node, and
+        must be pickleable by Ray cloudpickle.
+        """
+        if not isinstance(self, Catalog):
+            # self may contain the tuple returned from __reduce__ (ray pickle bug?)
+            if callable(self[0]) and isinstance(self[1], tuple):
+                logger.info(f"Invoking {self[0]} with positional args: {self[1]}")
+                return self[0](*self[1])
+            else:
+                err_msg = f"Expected `self` to be {Catalog}, but found: {self}"
+                raise RuntimeError(err_msg)
+
+        self._config = config
         self._impl = impl
-        self._impl.initialize(*args, **kwargs)
+        self._inner = self._impl.initialize(config=config, *args, **kwargs)
+        self._args = args
+        self._kwargs = kwargs
+
+    @property
+    def config(self):
+        return self._config
 
     @property
     def impl(self):
         return self._impl
 
+    @property
+    def inner(self) -> Optional[Any]:
+        return self._inner
+
+    # support pickle, copy, deepcopy, etc.
+    def __reduce__(self):
+        # instantiated catalogs may fail to pickle, so exclude _inner
+        # (e.g. Iceberg catalog w/ unserializable SSLContext from boto3 client)
+        return partial(self.__class__, **self._kwargs), (
+            self._config,
+            self._impl,
+            *self._args,
+        )
+
+    def __str__(self):
+        string_rep = f"{self.__class__.__name__}("
+        if self._args:
+            string_rep += f"args={self._args}, "
+        if self._kwargs:
+            string_rep += f"kwargs={self._kwargs}, "
+        if self._inner:
+            string_rep += f"inner={self._inner})"
+        return string_rep
+
+    def __repr__(self):
+        return self.__str__()
+
 
 @ray.remote
 class Catalogs:
     def __init__(
         self,
-        catalogs: Dict[str, Catalog],
-        default_catalog_name: str = None,
-        *args,
-        **kwargs,
+        catalogs: Union[Catalog, Dict[str, Catalog]],
+        default: Optional[str] = None,
     ):
-        if default_catalog_name and default_catalog_name not in catalogs:
-            raise ValueError(
-                f"Catalog {default_catalog_name} not found "
-                f"in catalogs to register: {catalogs}"
-            )
-        if not catalogs:
-            raise ValueError(
-                f"No catalogs given to register. "
-                f"Please specify one or more catalogs."
-            )
-        self.catalogs: Dict[str, Catalog] = catalogs
-        if default_catalog_name:
-            self.default_catalog = self.catalogs[default_catalog_name]
-        elif len(catalogs) == 1:
-            self.default_catalog = list(self.catalogs.values())[0]
-        else:
-            self.default_catalog = None
+        self._catalogs = {}
+        self._default_catalog_name = None
+        self._default_catalog = None
+        self.update(catalogs, default)
 
     def all(self) -> Dict[str, Catalog]:
-        return self.catalogs
+        return self._catalogs
+
+    def update(
+        self,
+        catalogs: Union[Catalog, Dict[str, Catalog]],
+        default: Optional[str] = None,
+    ) -> None:
+        if isinstance(catalogs, Catalog):
+            catalogs = {DEFAULT_CATALOG: catalogs}
+        elif not isinstance(catalogs, dict):
+            raise ValueError(f"Expected Catalog or dict, but found: {catalogs}")
+        self._catalogs.update(catalogs)
+        if default:
+            if default not in catalogs:
+                raise ValueError(
+                    f"Default catalog `{default}` not found in: {catalogs}"
+                )
+            self._default_catalog = self._catalogs[default]
+            self._default_catalog_name = default
+        elif len(catalogs) == 1:
+            self._default_catalog = list(self._catalogs.values())[0]
+        else:
+            self._default_catalog = None
 
     def names(self) -> List[str]:
-        return list(self.catalogs.keys())
+        return list(self._catalogs.keys())
+
+    def put(self, name: str, catalog: Catalog, set_default: bool = False) -> None:
+        self._catalogs[name] = catalog
+        if set_default or len(self._catalogs) == 1:
+            self._default_catalog = catalog
+
+    def get(self, name) -> Optional[Catalog]:
+        return self._catalogs.get(name)
 
-    def put(self, name: str, catalog: Catalog) -> None:
-        self.catalogs[name] = catalog
+    def pop(self, name) -> Optional[Catalog]:
+        catalog = self._catalogs.pop(name, None)
+        if catalog and self._default_catalog_name == name:
+            if len(self._catalogs) == 1:
+                self._default_catalog = list(self._catalogs.values())[0]
+            else:
+                self._default_catalog = None
+        return catalog
 
-    def get(self, name) -> Catalog:
-        return self.catalogs.get(name)
+    def clear(self) -> None:
+        self._catalogs.clear()
+        self._default_catalog = None
 
     def default(self) -> Optional[Catalog]:
-        return self.default_catalog
+        return self._default_catalog
 
 
-def init(
-    catalogs: Dict[str, Catalog],
-    default_catalog_name: str = None,
-    ray_init_args: Dict[str, Any] = None,
-    *args,
-    **kwargs,
-) -> None:
+def is_initialized(*args, **kwargs) -> bool:
+    """
+    Check if DeltaCAT is initialized.
+    """
+    global all_catalogs
 
     if not ray.is_initialized():
-        if ray_init_args:
-            ray.init(**ray_init_args)
-        else:
-            ray.init(address="auto")
+        # Any existing Catalogs actor reference must be stale - reset it
+        all_catalogs = None
+    return all_catalogs is not None
+
+
+def raise_if_not_initialized(
+    err_msg: str = "DeltaCAT is not initialized. Please call `deltacat.init()` and try again.",
+) -> None:
+    """
+    Raises a RuntimeError with the given error message if DeltaCAT is not
+    initialized.
+
+    :param err_msg: Custom error message to raise if DeltaCAT is not
+    initialized. If unspecified, the default error message is used.
+    """
+    if not is_initialized():
+        raise RuntimeError(err_msg)
+
+
+def init(
+    catalogs: Union[Dict[str, Catalog], Catalog] = {},
+    default: Optional[str] = None,
+    ray_init_args: Dict[str, Any] = {},
+    *,
+    force=False,
+) -> Optional[ray.runtime.BaseContext]:
+    """
+    Initialize DeltaCAT catalogs.
 
+    :param catalogs: A single Catalog instance or a map of catalog names to
+        Catalog instances.
+    :param default: The name of the default Catalog. If only one Catalog is
+        provided, it will always be the default.
+    :param ray_init_args: Keyword arguments to pass to `ray.init()`.
+    :param force: Whether to force DeltaCAT reinitialization. If True, reruns
+        ray.init(**ray_init_args) and overwrites all previously registered
+        catalogs.
+    :returns: The Ray context object if Ray was initialized, otherwise None.
+    """
     global all_catalogs
-    all_catalogs = Catalogs.remote(
-        catalogs=catalogs, default_catalog_name=default_catalog_name
+
+    if is_initialized() and not force:
+        logger.warning("DeltaCAT already initialized.")
+        return None
+
+    # initialize ray (and ignore reinitialization errors)
+    ray_init_args["ignore_reinit_error"] = True
+    context = ray.init(**ray_init_args)
+
+    # register custom serializer for catalogs since these may contain
+    # unserializable objects like boto3 clients with SSLContext
+    ray.util.register_serializer(
+        Catalog, serializer=Catalog.__reduce__, deserializer=Catalog.__init__
+    )
+    # TODO(pdames): If no catalogs are provided then re-initialize DeltaCAT
+    #  with all catalogs from the last session
+    all_catalogs = Catalogs.remote(catalogs=catalogs, default=default)
+    return context
+
+
+def init_local(
+    path: Optional[str] = None,
+    ray_init_args: Dict[str, Any] = {},
+    *,
+    force=False,
+) -> Optional[ray.runtime.BaseContext]:
+    """
+    Initialize DeltaCAT with a default local catalog.
+
+    This is a convenience function that creates a default catalog for local usage.
+    Equivalent to calling init(catalogs={"default": Catalog()}).
+
+    :param path: Optional path for catalog root directory. If not provided, uses
+        the default behavior of CatalogProperties (DELTACAT_ROOT env var or
+        "./.deltacat/").
+    :param ray_init_args: Keyword arguments to pass to `ray.init()`.
+    :param force: Whether to force DeltaCAT reinitialization. If True, reruns
+        ray.init(**ray_init_args) and overwrites all previously registered
+        catalogs.
+    :returns: The Ray context object if Ray was initialized, otherwise None.
+    """
+    from deltacat.catalog.model.properties import CatalogProperties
+
+    config = CatalogProperties(root=path) if path is not None else None
+    return init(
+        catalogs={"default": Catalog(config=config)},
+        default="default",
+        ray_init_args=ray_init_args,
+        force=force,
     )
+
+
+def get_catalog(name: Optional[str] = None) -> Catalog:
+    """
+    Get a catalog by name, or the default catalog if no name is provided.
+
+    Args:
+        name: Name of catalog to retrieve (optional, uses default if not provided)
+
+    Returns:
+        The requested Catalog, or ValueError if it does not exist
+    """
+    global all_catalogs
+
+    if not all_catalogs:
+        raise ValueError(
+            "No catalogs available! Call "
+            "`deltacat.init(catalogs={...})` to register one or more "
+            "catalogs then retry."
+        )
+    if name is not None:
+        catalog = ray.get(all_catalogs.get.remote(name))
+        if not catalog:
+            available_catalogs = ray.get(all_catalogs.all.remote()).values()
+            raise ValueError(
+                f"Catalog '{name}' not found. Available catalogs: "
+                f"{available_catalogs}."
+            )
+    else:
+        catalog = ray.get(all_catalogs.default.remote())
+        if not catalog:
+            available_catalogs = list(ray.get(all_catalogs.all.remote()).keys())
+            raise ValueError(
+                f"Call to get_catalog without name set failed because there "
+                f"is no default Catalog set. Available catalogs: "
+                f"{available_catalogs}."
+            )
+    return catalog
+
+
+def clear_catalogs() -> None:
+    """
+    Clear all catalogs from the global map of named catalogs.
+    """
+    if all_catalogs:
+        ray.get(all_catalogs.clear.remote())
+
+
+def pop_catalog(name: str) -> Optional[Catalog]:
+    """
+    Remove a named catalog from the global map of named catalogs.
+
+    Args:
+        name: Name of the catalog to remove.
+
+    Returns:
+        The removed catalog, or None if not found.
+    """
+    global all_catalogs
+
+    if not all_catalogs:
+        return None
+    catalog = ray.get(all_catalogs.pop.remote(name))
+    return catalog
+
+
+def put_catalog(
+    name: str,
+    catalog: Catalog = None,
+    *,
+    default: bool = False,
+    ray_init_args: Dict[str, Any] = {},
+    fail_if_exists: bool = False,
+    **kwargs,
+) -> Catalog:
+    """
+    Add a named catalog to the global map of named catalogs. Initializes
+    DeltaCAT if not already initialized.
+
+    Args:
+        name: Name of the catalog.
+        catalog: Catalog instance to use. If none is provided, then all
+            additional keyword arguments will be forwarded to
+            `CatalogProperties` for a default DeltaCAT native Catalog.
+        default:  Make this the default catalog if multiple catalogs are
+            available. If only one catalog is available, it will always be the
+            default.
+        ray_init_args: Ray initialization args (used only if ray is not already
+            initialized).
+        fail_if_exists: if True, raises an error if a catalog with the given
+            name already exists. If False, inserts or replaces the given
+            catalog name.
+        kwargs: Additional keyword arguments to forward to `CatalogProperties`
+            for a default DeltaCAT native Catalog.
+
+    Returns:
+        The catalog put in the named catalog map.
+    """
+    global all_catalogs
+
+    if not catalog:
+        catalog = Catalog(**kwargs)
+    if name is None:
+        raise ValueError("Catalog name cannot be None")
+
+    # Initialize, if necessary
+    if not is_initialized():
+        # We are initializing a single catalog - make it the default
+        if not default:
+            logger.info(
+                f"Calling put_catalog with set_as_default=False, "
+                f"but still setting Catalog {catalog} as default since it is "
+                f"the only catalog."
+            )
+        init({name: catalog}, ray_init_args=ray_init_args)
+        return catalog
+
+    # Fail if fail_if_exists and catalog already exists
+    if fail_if_exists:
+        try:
+            get_catalog(name)
+            # If we get here, catalog exists - raise error
+            raise ValueError(
+                f"Failed to put catalog {name} because it already exists and "
+                f"fail_if_exists={fail_if_exists}"
+            )
+        except ValueError as e:
+            if "not found" not in str(e):
+                # Re-raise if it's not a "catalog not found" error
+                raise
+            # If catalog doesn't exist, continue normally
+            pass
+
+    # Add the catalog (which may overwrite existing if fail_if_exists=False)
+    ray.get(all_catalogs.put.remote(name, catalog, default))
+    return catalog

deltacat/catalog/model/properties.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+from typing import Optional, Any
+import urllib.parse
+
+import os
+
+import pyarrow
+from deltacat.constants import DELTACAT_ROOT
+
+from deltacat.utils.filesystem import resolve_path_and_filesystem
+
+
+def get_catalog_properties(
+    *,
+    catalog: Optional[CatalogProperties] = None,
+    inner: Optional[CatalogProperties] = None,
+    **kwargs,
+) -> CatalogProperties:
+    """
+    Helper function to fetch CatalogProperties instance.
+
+    This will look first look for CatalogProperties in either "catalog"
+    or "inner" and otherwise passes all keyword arguments to the
+    CatalogProperties constructor.
+    """
+    properties = catalog if catalog is not None else inner
+    if properties is not None and isinstance(properties, CatalogProperties):
+        return properties
+    elif properties is not None and not isinstance(properties, CatalogProperties):
+        raise ValueError(
+            f"Expected catalog properties of type {CatalogProperties.__name__} "
+            f"but found {type(properties)}."
+        )
+    else:
+        return CatalogProperties(**kwargs)
+
+
+class CatalogProperties:
+    """
+    DeltaCAT catalog properties used to deterministically resolve a durable
+    DeltaCAT catalog instance. Properties are set from system environment
+    variables unless explicit overrides are provided during initialization.
+
+    Catalog and storage APIs rely on the property catalog to retrieve durable
+    state about the catalog they're working against.
+
+    Attributes:
+        root: The root path for catalog metadata and data storage. Resolved by
+            searching for the root path in the following order:
+            1. "root" constructor input argument
+            2. "DELTACAT_ROOT" system environment variable
+            3. default to "./.deltacat/"
+
+        filesystem: The filesystem implementation that should be used for
+            reading/writing files. If None, a filesystem will be inferred from
+            the catalog root path.
+
+        storage: Storage class implementation (overrides default filesystem
+            storage impl)
+    """
+
+    def __init__(
+        self,
+        root: Optional[str] = None,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+        storage=None,
+    ):
+        """
+        Initialize a CatalogProperties instance.
+
+        Args:
+            root: Catalog root directory path. Uses the "DELTACAT_ROOT"
+                system environment variable if not set, and defaults to
+                "./.deltacat/" if this environment variable is not set.
+            filesystem: The filesystem implementation that should be used for
+                reading these files. If None, a filesystem will be inferred.
+                If provided, this will be validated for compatibility with the
+                catalog root path.
+            storage: DeltaCAT storage implementation override.
+        """
+        # set root, using precedence rules described in pydoc
+        if root is None:
+            # Check environment variables
+            root = DELTACAT_ROOT
+            if not root:
+                # Default to "./.deltacat/"
+                root = os.path.join(os.getcwd(), ".deltacat")
+
+        # Store the original root with its scheme for reconstruction later
+        self._original_root = root
+        self._original_scheme = urllib.parse.urlparse(root).scheme
+
+        resolved_root, resolved_filesystem = resolve_path_and_filesystem(
+            path=root,
+            filesystem=filesystem,
+        )
+        self._root = resolved_root
+        self._filesystem = resolved_filesystem
+        self._storage = storage
+
+    @property
+    def root(self) -> str:
+        return self._root
+
+    @property
+    def filesystem(self) -> Optional[pyarrow.fs.FileSystem]:
+        return self._filesystem
+
+    @property
+    def storage(self) -> Optional[Any]:
+        """
+        Return overridden storage impl, if any
+        """
+        return self._storage
+
+    def reconstruct_full_path(self, path: str) -> str:
+        """
+        Reconstruct a full path with the original scheme for external readers.
+
+        This addresses GitHub issue #567 by ensuring that cloud storage URIs
+        include the relevant scheme prefix (e.g., s3://) that some file readers
+        require regardless of the filesystem being used to read the file
+        (e.g., Daft).
+
+        Args:
+            path: A path relative to the catalog root or absolute path
+
+        Returns:
+            Full path with appropriate scheme prefix for external readers
+        """
+        # If the path already has a scheme, return it as-is
+        if urllib.parse.urlparse(path).scheme:
+            return path
+
+        # If we don't have an original scheme (local filesystem), return as-is
+        if not self._original_scheme:
+            return path
+
+        # Reconstruct the full path with the original scheme
+        # Handle both absolute and relative paths
+        if path.startswith("/"):
+            # Absolute path - this shouldn't happen normally but handle it
+            return f"{self._original_scheme}:/{path}"
+        else:
+            # Relative path - prepend the s3:// scheme
+            return f"{self._original_scheme}://{path}"
+
+    def __str__(self):
+        return (
+            f"{self.__class__.__name__}(root={self.root}, filesystem={self.filesystem})"
+        )
+
+    def __repr__(self):
+        return self.__str__()

deltacat/catalog/model/table_definition.py

@@ -1,19 +1,30 @@
 # Allow self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
+from typing import Optional, Any
+
 from deltacat.storage import Stream, Table, TableVersion
+from deltacat.storage.model.scan.push_down import Pushdown
+from deltacat.storage.model.scan.scan_plan import ScanPlan
+from deltacat.storage.util.scan_planner import ScanPlanner
 
 
 class TableDefinition(dict):
     @staticmethod
     def of(
-        table: Table, table_version: TableVersion, stream: Stream
+        table: Table,
+        table_version: TableVersion,
+        stream: Stream,
+        native_object: Optional[Any] = None,
+        scan_planner: Optional[ScanPlanner] = None,
     ) -> TableDefinition:
         return TableDefinition(
             {
                 "table": table,
                 "tableVersion": table_version,
                 "stream": stream,
+                "nativeObject": native_object,
+                "scan_planner": scan_planner,
             }
         )
 
@@ -28,3 +39,23 @@ class TableDefinition(dict):
     @property
     def stream(self) -> Stream:
         return self["stream"]
+
+    @property
+    def native_object(self) -> Optional[Any]:
+        return self.get("nativeObject")
+
+    @property
+    def scan_planner(self) -> Optional[ScanPlanner]:
+        return self.get("scan_planner")
+
+    def create_scan_plan(self, pushdown: Optional[Pushdown] = None) -> ScanPlan:
+        if not self.scan_planner:
+            raise RuntimeError(
+                f"ScanPlanner is not initialized for table '{self.table.table_name}' "
+                f"of namespace '{self.table.namespace}'"
+            )
+        return self.scan_planner.create_scan_plan(
+            table_name=self.table.table_name,
+            namespace=self.table.namespace,
+            pushdown=pushdown,
+        )

deltacat/compute/__init__.py

@@ -0,0 +1,14 @@
+from deltacat.compute.jobs.client import (
+    DeltaCatJobClient,
+    job_client,
+    local_job_client,
+)
+
+from ray.job_submission import JobStatus
+
+__all__ = [
+    "job_client",
+    "local_job_client",
+    "DeltaCatJobClient",
+    "JobStatus",
+]