deltacat 2.0.0b11__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/catalog/model/catalog.py

@@ -127,7 +127,7 @@ class Catalogs:
 
     def put(self, name: str, catalog: Catalog, set_default: bool = False) -> None:
         self._catalogs[name] = catalog
-        if set_default:
+        if set_default or len(self._catalogs) == 1:
             self._default_catalog = catalog
 
     def get(self, name) -> Optional[Catalog]:
@@ -182,7 +182,7 @@ def init(
     ray_init_args: Dict[str, Any] = {},
     *,
     force=False,
-) -> None:
+) -> Optional[ray.runtime.BaseContext]:
     """
     Initialize DeltaCAT catalogs.
 
@@ -194,16 +194,17 @@ def init(
     :param force: Whether to force DeltaCAT reinitialization. If True, reruns
         ray.init(**ray_init_args) and overwrites all previously registered
         catalogs.
+    :returns: The Ray context object if Ray was initialized, otherwise None.
     """
     global all_catalogs
 
     if is_initialized() and not force:
         logger.warning("DeltaCAT already initialized.")
-        return
+        return None
 
     # initialize ray (and ignore reinitialization errors)
     ray_init_args["ignore_reinit_error"] = True
-    ray.init(**ray_init_args)
+    context = ray.init(**ray_init_args)
 
     # register custom serializer for catalogs since these may contain
     # unserializable objects like boto3 clients with SSLContext
@@ -213,6 +214,39 @@ def init(
     # TODO(pdames): If no catalogs are provided then re-initialize DeltaCAT
     #  with all catalogs from the last session
     all_catalogs = Catalogs.remote(catalogs=catalogs, default=default)
+    return context
+
+
+def init_local(
+    path: Optional[str] = None,
+    ray_init_args: Dict[str, Any] = {},
+    *,
+    force=False,
+) -> Optional[ray.runtime.BaseContext]:
+    """
+    Initialize DeltaCAT with a default local catalog.
+
+    This is a convenience function that creates a default catalog for local usage.
+    Equivalent to calling init(catalogs={"default": Catalog()}).
+
+    :param path: Optional path for catalog root directory. If not provided, uses
+        the default behavior of CatalogProperties (DELTACAT_ROOT env var or
+        "./.deltacat/").
+    :param ray_init_args: Keyword arguments to pass to `ray.init()`.
+    :param force: Whether to force DeltaCAT reinitialization. If True, reruns
+        ray.init(**ray_init_args) and overwrites all previously registered
+        catalogs.
+    :returns: The Ray context object if Ray was initialized, otherwise None.
+    """
+    from deltacat.catalog.model.properties import CatalogProperties
+
+    config = CatalogProperties(root=path) if path is not None else None
+    return init(
+        catalogs={"default": Catalog(config=config)},
+        default="default",
+        ray_init_args=ray_init_args,
+        force=force,
+    )
 
 
 def get_catalog(name: Optional[str] = None) -> Catalog:
@@ -244,7 +278,7 @@ def get_catalog(name: Optional[str] = None) -> Catalog:
     else:
         catalog = ray.get(all_catalogs.default.remote())
         if not catalog:
-            available_catalogs = ray.get(all_catalogs.all.remote()).values()
+            available_catalogs = list(ray.get(all_catalogs.all.remote()).keys())
             raise ValueError(
                 f"Call to get_catalog without name set failed because there "
                 f"is no default Catalog set. Available catalogs: "
@@ -334,12 +368,17 @@ def put_catalog(
     if fail_if_exists:
         try:
             get_catalog(name)
-        except ValueError:
+            # If we get here, catalog exists - raise error
+            raise ValueError(
+                f"Failed to put catalog {name} because it already exists and "
+                f"fail_if_exists={fail_if_exists}"
+            )
+        except ValueError as e:
+            if "not found" not in str(e):
+                # Re-raise if it's not a "catalog not found" error
+                raise
+            # If catalog doesn't exist, continue normally
             pass
-        raise ValueError(
-            f"Failed to put catalog {name} because it already exists and "
-            f"fail_if_exists={fail_if_exists}"
-        )
 
     # Add the catalog (which may overwrite existing if fail_if_exists=False)
     ray.get(all_catalogs.put.remote(name, catalog, default))

deltacat/catalog/model/properties.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from typing import Optional, Any
+import urllib.parse
 
 import os
 
@@ -76,6 +77,7 @@ class CatalogProperties:
                 reading these files. If None, a filesystem will be inferred.
                 If provided, this will be validated for compatibility with the
                 catalog root path.
+            storage: DeltaCAT storage implementation override.
         """
         # set root, using precedence rules described in pydoc
         if root is None:
@@ -85,6 +87,10 @@ class CatalogProperties:
                 # Default to "./.deltacat/"
                 root = os.path.join(os.getcwd(), ".deltacat")
 
+        # Store the original root with its scheme for reconstruction later
+        self._original_root = root
+        self._original_scheme = urllib.parse.urlparse(root).scheme
+
         resolved_root, resolved_filesystem = resolve_path_and_filesystem(
             path=root,
             filesystem=filesystem,
@@ -108,6 +114,38 @@ class CatalogProperties:
         """
         return self._storage
 
+    def reconstruct_full_path(self, path: str) -> str:
+        """
+        Reconstruct a full path with the original scheme for external readers.
+
+        This addresses GitHub issue #567 by ensuring that cloud storage URIs
+        include the relevant scheme prefix (e.g., s3://) that some file readers
+        require regardless of the filesystem being used to read the file
+        (e.g., Daft).
+
+        Args:
+            path: A path relative to the catalog root or absolute path
+
+        Returns:
+            Full path with appropriate scheme prefix for external readers
+        """
+        # If the path already has a scheme, return it as-is
+        if urllib.parse.urlparse(path).scheme:
+            return path
+
+        # If we don't have an original scheme (local filesystem), return as-is
+        if not self._original_scheme:
+            return path
+
+        # Reconstruct the full path with the original scheme
+        # Handle both absolute and relative paths
+        if path.startswith("/"):
+            # Absolute path - this shouldn't happen normally but handle it
+            return f"{self._original_scheme}:/{path}"
+        else:
+            # Relative path - prepend the s3:// scheme
+            return f"{self._original_scheme}://{path}"
+
     def __str__(self):
         return (
             f"{self.__class__.__name__}(root={self.root}, filesystem={self.filesystem})"

deltacat/compute/compactor/compaction_session.py

@@ -5,7 +5,8 @@ import logging
 import ray
 import time
 import json
-from deltacat.aws import s3u as s3_utils
+import posixpath
+from deltacat.utils.filesystem import resolve_path_and_filesystem
 import deltacat
 from deltacat import logs
 import pyarrow as pa
@@ -25,7 +26,7 @@ from deltacat.storage import (
     DeltaLocator,
     Partition,
     PartitionLocator,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat.compute.compactor.model.compact_partition_params import (
     CompactPartitionParams,
@@ -40,7 +41,7 @@ from deltacat.compute.compactor.steps import dedupe as dd
 from deltacat.compute.compactor.steps import hash_bucket as hb
 from deltacat.compute.compactor.steps import materialize as mat
 from deltacat.compute.compactor.utils import io
-from deltacat.compute.compactor.utils import round_completion_file as rcf
+from deltacat.compute.compactor.utils import round_completion_reader as rci
 
 from deltacat.types.media import ContentType
 from deltacat.utils.placement import PlacementGroupConfig
@@ -65,13 +66,37 @@ DEFAULT_DEDUPE_MAX_PARALLELISM_RATIO_ARG: int = 1
 DEFAULT_PROPERTIES_ARG: Dict[str, Any] = {}
 
 
+def _upload_audit_data(url: str, content: str, **kwargs) -> None:
+    """
+    Upload audit data to the specified URL using filesystem-agnostic operations.
+    """
+    try:
+        path, filesystem = resolve_path_and_filesystem(url)
+
+        # Create parent directories if they don't exist
+        parent_dir = posixpath.dirname(path)
+        if parent_dir:
+            try:
+                filesystem.create_dir(parent_dir, recursive=True)
+            except Exception as dir_error:
+                # Directory might already exist, which is fine
+                logger.debug(
+                    f"Directory creation warning for {parent_dir}: {dir_error}"
+                )
+
+        with filesystem.open_output_stream(path) as stream:
+            stream.write(content.encode("utf-8"))
+    except Exception as e:
+        logger.warning(f"Failed to upload audit data to {url}: {e}")
+
+
 def check_preconditions(
     source_partition_locator: PartitionLocator,
     destination_partition_locator: PartitionLocator,
     sort_keys: List[SortKey],
     max_records_per_output_file: int,
     new_hash_bucket_count: Optional[int],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> int:
@@ -104,7 +129,7 @@ def compact_partition(
     source_partition_locator: PartitionLocator,
     destination_partition_locator: PartitionLocator,
     primary_keys: Set[str],
-    compaction_artifact_s3_bucket: str,
+    compaction_artifact_path: str,
     last_stream_position_to_compact: int,
     *,
     hash_bucket_count: Optional[int] = None,
@@ -123,37 +148,29 @@ def compact_partition(
     metrics_config: Optional[MetricsConfig] = None,
     list_deltas_kwargs: Optional[Dict[str, Any]] = None,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     object_store: Optional[IObjectStore] = RayPlasmaObjectStore(),
-    s3_client_kwargs: Optional[Dict[str, Any]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
-) -> Optional[str]:
+) -> None:
     if deltacat_storage_kwargs is None:
         deltacat_storage_kwargs = {}
     if not importlib.util.find_spec("memray"):
         logger.info(f"memray profiler not available, disabling all profiling")
         enable_profiler = False
 
-    if s3_client_kwargs is None:
-        s3_client_kwargs = {}
-
     # memray official documentation link:
     # https://bloomberg.github.io/memray/getting_started.html
     with memray.Tracker(
         f"compaction_partition.bin"
     ) if enable_profiler else nullcontext():
         partition = None
-        (
-            new_partition,
-            new_rci,
-            new_rcf_partition_locator,
-        ) = _execute_compaction_round(
+        (new_partition, new_rci,) = _execute_compaction_round(
             source_partition_locator,
             destination_partition_locator,
             primary_keys,
-            compaction_artifact_s3_bucket,
+            compaction_artifact_path,
             last_stream_position_to_compact,
             hash_bucket_count,
             sort_keys,
@@ -169,9 +186,8 @@ def compact_partition(
             metrics_config,
             list_deltas_kwargs,
             read_kwargs_provider,
-            s3_table_writer_kwargs,
+            table_writer_kwargs,
             object_store,
-            s3_client_kwargs,
             deltacat_storage,
             deltacat_storage_kwargs,
             **kwargs,
@@ -182,30 +198,23 @@ def compact_partition(
         logger.info(
             f"Partition-{source_partition_locator.partition_values}-> Compaction session data processing completed"
         )
-        round_completion_file_s3_url = None
         if partition:
             logger.info(f"Committing compacted partition to: {partition.locator}")
+            # Set the round completion info on the partition before committing
+            partition.compaction_round_completion_info = new_rci
             partition = deltacat_storage.commit_partition(
-                partition, **deltacat_storage_kwargs
+                partition,
+                **deltacat_storage_kwargs,
             )
             logger.info(f"Committed compacted partition: {partition}")
-
-            round_completion_file_s3_url = rcf.write_round_completion_file(
-                compaction_artifact_s3_bucket,
-                new_rcf_partition_locator,
-                partition.locator,
-                new_rci,
-                **s3_client_kwargs,
-            )
         logger.info(f"Completed compaction session for: {source_partition_locator}")
-        return round_completion_file_s3_url
 
 
 def _execute_compaction_round(
     source_partition_locator: PartitionLocator,
     destination_partition_locator: PartitionLocator,
     primary_keys: Set[str],
-    compaction_artifact_s3_bucket: str,
+    compaction_artifact_path: str,
     last_stream_position_to_compact: int,
     hash_bucket_count: Optional[int],
     sort_keys: List[SortKey],
@@ -221,24 +230,25 @@ def _execute_compaction_round(
     metrics_config: Optional[MetricsConfig],
     list_deltas_kwargs: Optional[Dict[str, Any]],
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    s3_table_writer_kwargs: Optional[Dict[str, Any]],
+    table_writer_kwargs: Optional[Dict[str, Any]],
     object_store: Optional[IObjectStore],
-    s3_client_kwargs: Optional[Dict[str, Any]],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
-) -> Tuple[Optional[Partition], Optional[RoundCompletionInfo], Optional[str]]:
+) -> Tuple[Optional[Partition], Optional[RoundCompletionInfo]]:
     if deltacat_storage_kwargs is None:
         deltacat_storage_kwargs = {}
-    rcf_source_partition_locator = (
+    rci_source_partition_locator = (
         rebase_source_partition_locator
         if rebase_source_partition_locator
         else source_partition_locator
     )
-    base_audit_url = rcf_source_partition_locator.path(
-        f"s3://{compaction_artifact_s3_bucket}/compaction-audit"
+    # Construct audit URL using filesystem-agnostic path joining
+    audit_url = posixpath.join(
+        compaction_artifact_path,
+        "compaction-audit.json",
+        f"{rci_source_partition_locator.hexdigest()}.json",
     )
-    audit_url = f"{base_audit_url}.json"
 
     logger.info(f"Compaction audit will be written to {audit_url}")
 
@@ -312,11 +322,11 @@ def _execute_compaction_round(
     # read the results from any previously completed compaction round
     round_completion_info = None
     if not rebase_source_partition_locator:
-        round_completion_info = rcf.read_round_completion_file(
-            compaction_artifact_s3_bucket,
-            source_partition_locator,
-            destination_partition_locator,
-            **s3_client_kwargs,
+        round_completion_info = rci.read_round_completion_info(
+            source_partition_locator=source_partition_locator,
+            destination_partition_locator=destination_partition_locator,
+            deltacat_storage=deltacat_storage,
+            deltacat_storage_kwargs=deltacat_storage_kwargs,
         )
         if not round_completion_info:
             logger.info(
@@ -363,15 +373,11 @@ def _execute_compaction_round(
         delta_discovery_end - delta_discovery_start
     )
 
-    s3_utils.upload(
-        compaction_audit.audit_url,
-        str(json.dumps(compaction_audit)),
-        **s3_client_kwargs,
-    )
+    _upload_audit_data(audit_url, json.dumps(compaction_audit))
 
     if not input_deltas:
         logger.info("No input deltas found to compact.")
-        return None, None, None
+        return None, None
 
     # limit the input deltas to fit on this cluster and convert them to
     # annotated deltas of equivalent size for easy parallel distribution
@@ -464,11 +470,7 @@ def _execute_compaction_round(
         hb_end - hb_start,
     )
 
-    s3_utils.upload(
-        compaction_audit.audit_url,
-        str(json.dumps(compaction_audit)),
-        **s3_client_kwargs,
-    )
+    _upload_audit_data(audit_url, json.dumps(compaction_audit))
 
     all_hash_group_idx_to_obj_id = defaultdict(list)
     for hb_result in hb_results:
@@ -485,9 +487,9 @@ def _execute_compaction_round(
     )
 
     compaction_audit.set_input_records(total_hb_record_count.item())
-    # TODO (pdames): when resources are freed during the last round of hash
-    #  bucketing, start running dedupe tasks that read existing dedupe
-    #  output from S3 then wait for hash bucketing to finish before continuing
+    # TODO(pdames): when resources are freed during the last round of hash bucketing,
+    #  start running dedupe tasks that read hash bucket output from storage then
+    #  wait for hash bucketing to finish before continuing
 
     # create a new stream for this round
     compacted_stream_locator = destination_partition_locator.stream_locator
@@ -497,6 +499,7 @@ def _execute_compaction_round(
         compacted_stream_locator.table_version,
         **deltacat_storage_kwargs,
     )
+
     partition = deltacat_storage.stage_partition(
         stream,
         destination_partition_locator.partition_values,
@@ -571,9 +574,9 @@ def _execute_compaction_round(
     logger.info(f"Materialize buckets created: " f"{len(all_mat_buckets_to_obj_id)}")
 
     compaction_audit.set_materialize_buckets(len(all_mat_buckets_to_obj_id))
-    # TODO(pdames): when resources are freed during the last round of deduping
+    # TODO(pdames): when resources are freed during the last round of deduping,
     #  start running materialize tasks that read materialization source file
-    #  tables from S3 then wait for deduping to finish before continuing
+    #  tables from storage then wait for deduping to finish before continuing
 
     # TODO(pdames): balance inputs to materialization tasks to ensure that each
     #  task has an approximately equal amount of input to materialize
@@ -584,11 +587,7 @@ def _execute_compaction_round(
     # parallel step 3:
     # materialize records to keep by index
 
-    s3_utils.upload(
-        compaction_audit.audit_url,
-        str(json.dumps(compaction_audit)),
-        **s3_client_kwargs,
-    )
+    _upload_audit_data(audit_url, json.dumps(compaction_audit))
 
     materialize_start = time.monotonic()
     mat_tasks_pending = invoke_parallel(
@@ -610,7 +609,7 @@ def _execute_compaction_round(
         enable_profiler=enable_profiler,
         metrics_config=metrics_config,
         read_kwargs_provider=read_kwargs_provider,
-        s3_table_writer_kwargs=s3_table_writer_kwargs,
+        table_writer_kwargs=table_writer_kwargs,
         object_store=object_store,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,
@@ -693,11 +692,7 @@ def _execute_compaction_round(
         telemetry_time_hb + telemetry_time_dd + telemetry_time_materialize
     )
 
-    s3_utils.upload(
-        compaction_audit.audit_url,
-        str(json.dumps(compaction_audit)),
-        **s3_client_kwargs,
-    )
+    _upload_audit_data(audit_url, json.dumps(compaction_audit))
 
     new_round_completion_info = RoundCompletionInfo.of(
         last_stream_position_compacted,
@@ -710,6 +705,7 @@ def _execute_compaction_round(
         hash_bucket_count,
         None,
         CompactorVersion.V1.value,
+        prev_source_partition_locator=rci_source_partition_locator,
     )
 
     logger.info(
@@ -721,17 +717,43 @@ def _execute_compaction_round(
     return (
         partition,
         new_round_completion_info,
-        rcf_source_partition_locator,
     )
 
 
 def compact_partition_from_request(
     compact_partition_params: CompactPartitionParams,
     *compact_partition_pos_args,
-) -> Optional[str]:
+) -> None:
     """
     Wrapper for compact_partition that allows for the compact_partition parameters to be
     passed in as a custom dictionary-like CompactPartitionParams object along with any compact_partition positional arguments.
     :param compact_partition_params:
     """
-    return compact_partition(*compact_partition_pos_args, **compact_partition_params)
+    # Extract required positional arguments
+    source_partition_locator = compact_partition_params.source_partition_locator
+    destination_partition_locator = (
+        compact_partition_params.destination_partition_locator
+    )
+    primary_keys = compact_partition_params.primary_keys
+    compaction_artifact_path = compact_partition_params.compaction_artifact_path
+    last_stream_position_to_compact = (
+        compact_partition_params.last_stream_position_to_compact
+    )
+
+    # Create a copy of params without the positional arguments
+    kwargs_params = dict(compact_partition_params)
+    kwargs_params.pop("source_partition_locator", None)
+    kwargs_params.pop("destination_partition_locator", None)
+    kwargs_params.pop("primary_keys", None)
+    kwargs_params.pop("last_stream_position_to_compact", None)
+    # Don't pop compaction_artifact_path as it's a computed property, not stored in the dict
+
+    compact_partition(
+        source_partition_locator,
+        destination_partition_locator,
+        primary_keys,
+        compaction_artifact_path,
+        last_stream_position_to_compact,
+        *compact_partition_pos_args,
+        **kwargs_params,
+    )