deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/__init__.py

@@ -8,7 +8,12 @@ from deltacat.api import (
     list,
     put,
 )
-from deltacat.catalog.delegate import (
+from deltacat.storage.model.transaction import (
+    transaction,
+    transactions,
+    read_transaction,
+)
+from deltacat.catalog import (  # noqa: F401
     alter_namespace,
     alter_table,
     create_namespace,
@@ -27,22 +32,30 @@ from deltacat.catalog.delegate import (
     table_exists,
     truncate_table,
     write_to_table,
-)
-from deltacat.catalog.model.catalog import (  # noqa: F401
-    Catalog,
-    Catalogs,
-    raise_if_not_initialized,
-    is_initialized,
     init,
+    init_local,
+    is_initialized,
+    clear_catalogs,
     get_catalog,
+    get_catalog_properties,
+    pop_catalog,
     put_catalog,
+    raise_if_not_initialized,
+    Catalog,
+    CatalogProperties,
+    TableDefinition,
 )
-from deltacat.catalog.model.table_definition import TableDefinition
 from deltacat.compute import (
     job_client,
     local_job_client,
 )
 from deltacat.storage import (
+    BucketingStrategy,
+    BucketTransform,
+    BucketTransformParameters,
+    DayTransform,
+    HourTransform,
+    IdentityTransform,
     Dataset,
     DistributedDataset,
     Field,
@@ -58,22 +71,50 @@ from deltacat.storage import (
     SortKey,
     SortOrder,
     SortScheme,
+    TableProperties,
+    TransactionStatus,
+    Transform,
+    TransformName,
+    TransformParameters,
+    TruncateTransform,
+    TruncateTransformParameters,
+    TruncateStrategy,
+    UnknownTransform,
+    VoidTransform,
+    YearTransform,
     NullOrder,
 )
-from deltacat.storage.rivulet import Dataset as RivDataset, Datatype as RivDatatype
 from deltacat.types.media import (
     ContentEncoding,
     ContentType,
     DatasetType,
     DatastoreType,
 )
-
-from deltacat.types.tables import TableWriteMode
+from deltacat.types.tables import (
+    TableWriteMode,
+    TableProperty,
+    TableReadOptimizationLevel,
+    SchemaEvolutionMode,
+    from_pandas,
+    from_pyarrow,
+    from_manifest_table,
+    to_pyarrow,
+    to_pandas,
+    dataset_length,
+    dataset_size,
+    dataset_column_names,
+    dataset_schema,
+)
 from deltacat.utils.url import DeltaCatUrl
 
+write = write_to_table
+read = read_table
+
 __iceberg__ = []
 if importlib.util.find_spec("pyiceberg") is not None:
-    from deltacat.catalog.iceberg import impl as IcebergCatalog  # noqa: F401
+    from deltacat.experimental.catalog.iceberg import (  # noqa: F401
+        impl as IcebergCatalog,
+    )
 
     __iceberg__ = [
         "IcebergCatalog",
@@ -81,7 +122,7 @@ if importlib.util.find_spec("pyiceberg") is not None:
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "2.0.0b10"
+__version__ = "2.0.0b12"
 
 
 __all__ = [
@@ -92,6 +133,8 @@ __all__ = [
     "get",
     "list",
     "put",
+    "transaction",
+    "transactions",
     "alter_table",
     "create_table",
     "drop_table",
@@ -108,28 +151,50 @@ __all__ = [
     "create_namespace",
     "drop_namespace",
     "default_namespace",
+    "write",
     "write_to_table",
+    "read",
     "read_table",
+    "read_transaction",
+    "init",
+    "init_local",
+    "is_initialized",
+    "clear_catalogs",
     "get_catalog",
+    "get_catalog_properties",
+    "pop_catalog",
     "put_catalog",
     "raise_if_not_initialized",
-    "is_initialized",
-    "init",
+    "dataset_length",
+    "dataset_size",
+    "dataset_column_names",
+    "dataset_schema",
+    "from_pandas",
+    "from_pyarrow",
+    "from_manifest_table",
+    "to_pandas",
+    "to_pyarrow",
+    "BucketingStrategy",
+    "BucketTransform",
+    "BucketTransformParameters",
     "Catalog",
+    "CatalogProperties",
     "ContentType",
     "ContentEncoding",
     "Dataset",
     "DatasetType",
     "DatastoreType",
+    "DayTransform",
     "DeltaCatUrl",
     "DistributedDataset",
-    "RivDataset",
-    "RivDatatype",
     "Field",
+    "HourTransform",
+    "IdentityTransform",
     "LifecycleState",
     "ListResult",
     "LocalDataset",
     "LocalTable",
+    "MonthTransform",
     "Namespace",
     "NullOrder",
     "PartitionKey",
@@ -141,6 +206,20 @@ __all__ = [
     "SortScheme",
     "TableDefinition",
     "TableWriteMode",
+    "TableProperties",
+    "TableProperty",
+    "TableReadOptimizationLevel",
+    "SchemaEvolutionMode",
+    "TransactionStatus",
+    "Transform",
+    "TransformName",
+    "TransformParameters",
+    "TruncateTransform",
+    "TruncateTransformParameters",
+    "TruncateStrategy",
+    "UnknownTransform",
+    "VoidTransform",
+    "YearTransform",
 ]
 
 __all__ += __iceberg__

deltacat/api.py

@@ -1,6 +1,7 @@
 import time
 from dataclasses import dataclass
 from typing import Any, Union, List, Optional, Dict, Callable, Tuple
+import logging
 
 import ray
 import deltacat as dc
@@ -15,6 +16,12 @@ from deltacat.io import (
     DeltacatReadType,
 )
 from deltacat.storage import (
+    Namespace,
+    Table,
+    TableVersion,
+    Stream,
+    Partition,
+    Delta,
     Dataset,
     DistributedDataset,
     ListResult,
@@ -44,6 +51,9 @@ from deltacat.utils.ray_utils.runtime import (
     other_live_node_resource_keys,
     find_max_single_node_resource_type,
 )
+from deltacat import logs
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 """
     # CLI Example of Copying from Source to Dest without file conversion
@@ -72,38 +82,6 @@ from deltacat.utils.ray_utils.runtime import (
 """
 
 
-def _copy_dc(
-    source: DeltaCatUrl,
-    destination: DeltaCatUrl,
-    recursive: bool = False,
-) -> Metafile:
-    if recursive:
-        src_obj = list(source, recursive=True)
-    else:
-        src_obj = get(source) if not source.url.endswith("/*") else list(source)
-    """
-    dc_dest_url = DeltacatUrl(destination)
-    # TODO(pdames): Add writer with support for Ray Dataset DeltaCAT Sink &
-    #  Recursive DeltaCAT source object copies. Ideally, the Ray Dataset read
-    #  is lazy, and only indexes metadata about the objects at source instead
-    #  of eagerly converting them to PyArrow-based Blocks.
-    dc_dest_url.writer(src_obj, recursive=recursive)
-    """
-
-    src_parts = source.url.split("/")
-    src_parts = [part for part in src_parts if part]
-    dst_parts = destination.url.split("/")
-    dst_parts = [part for part in dst_parts if part]
-    dc.raise_if_not_initialized()
-    if len(src_parts) != len(dst_parts):
-        # TODO(pdames): Better error message.
-        raise ValueError(
-            f"Cannot copy {source} to {destination}. "
-            f"Source and destination must share the same type."
-        )
-    return put(destination, metafile=src_obj)
-
-
 def copy(
     src: DeltaCatUrl,
     dst: DeltaCatUrl,
@@ -123,6 +101,7 @@ def copy(
         "gz": 35,
         "bz2": 35,
         "zip": 35,
+        "zst": 35,
         "7z": 35,
         "*": 2.5,
     },
@@ -135,11 +114,10 @@ def copy(
     Copies data from the source datastore to the destination datastore. By
     default, this method launches one parallel Ray process to read/transform
     each input file found in the source followed by one parallel Ray process
-    to write each output file to the destination. Files written to the
-    destination are split or combined to contain uniform record counts. To
-    ensure that adequate resources are available to complete the operation,
-    you may optionally specify minimum cluster and/or worker CPUs to wait for
-    before starting parallel processing.
+    to write each output file to the destination. To ensure that adequate
+    resources are available to complete the operation, you may optionally
+    specify minimum cluster and/or worker CPUs to wait for before starting
+    parallel processing.
 
     Args:
         src: DeltaCAT URL of the source datastore to read.
@@ -190,6 +168,73 @@ def copy(
         )
 
 
+def _copy_objects_in_order(
+    src_objects: List[Metafile],
+    destination: DeltaCatUrl,
+) -> Union[Metafile, List[Metafile]]:
+    dc_dest_url = DeltaCatUrl(destination.url)
+    catalog_name = dc_dest_url.catalog_name
+
+    copied_results = []
+
+    # Group objects by type for hierarchical copying
+    # Copy objects in strict hierarchical order
+    # Namespace -> Table -> TableVersion -> Stream -> Partition -> Delta
+    ordered_objects_by_type = {
+        Namespace: [],
+        Table: [],
+        TableVersion: [],
+        Stream: [],
+        Partition: [],
+        Delta: [],
+    }
+
+    for obj in src_objects:
+        obj_class = Metafile.get_class(obj.to_serializable())
+        ordered_objects_by_type[obj_class].append(obj)
+
+    # TODO(pdames): Support copying uncommitted streams/partitions.
+    # TODO(pdames): Support parallel/distributed copies.
+    for obj_class, objects in ordered_objects_by_type.items():
+        if objects:
+            logger.info(f"Copying {len(objects)} {obj_class} objects...")
+        if obj_class == TableVersion:
+            # sort table versions by ascending table version
+            objects.sort(key=lambda x: x.current_version_number())
+        if obj_class == Delta:
+            # sort deltas by ascending stream position
+            objects.sort(key=lambda x: x.stream_position)
+        for i, obj in enumerate(objects):
+            logger.info(f"Copying object {i+1}/{len(objects)}: {obj.url}")
+            dest_url = DeltaCatUrl(obj.url(catalog_name=catalog_name))
+            logger.info(f"Destination URL for object {i+1}/{len(objects)}: {dest_url}")
+            result = put(dest_url, metafile=obj)
+            copied_results.append(result)
+            logger.info(f"Successfully copied object {i+1}/{len(objects)}")
+    return copied_results[0] if len(copied_results) == 1 else copied_results
+
+
+def _copy_dc(
+    source: DeltaCatUrl,
+    destination: DeltaCatUrl,
+    recursive: bool = False,
+) -> Union[Metafile, List[Metafile]]:
+    dc.raise_if_not_initialized()
+    if len(source.url.split("/")) != len(destination.url.split("/")):
+        # TODO(pdames): Better error message.
+        raise ValueError(
+            f"Cannot copy {source} to {destination}. "
+            f"Source and destination must share the same type."
+        )
+    if recursive:
+        src_objects = list(DeltaCatUrl(source.url.rstrip("/**")), recursive=True)
+    elif source.url.endswith("/*"):
+        src_objects = list(DeltaCatUrl(source.url.rstrip("/*")))
+    else:
+        src_objects = [get(source)]
+    return _copy_objects_in_order(src_objects, destination)
+
+
 def concat(source, destination):
     raise NotImplementedError
 
@@ -214,9 +259,13 @@ def _list_all_metafiles(
     metafiles: ListResult[Metafile] = lister(**kwargs)
     list_results.append(metafiles)
     if recursive:
+        # Process each level of the hierarchy
+        current_level_metafiles = [mf for mf in metafiles.all_items()]
+
         for lister, kwarg_name, kwarg_val_resolver_fn in reader.listers:
+            next_level_metafiles = []
             # each subsequent lister needs to inject missing keyword args from the parent metafile
-            for metafile in metafiles.all_items():
+            for metafile in current_level_metafiles:
                 kwargs_update = (
                     {kwarg_name: kwarg_val_resolver_fn(metafile)}
                     if kwarg_name and kwarg_val_resolver_fn
@@ -226,8 +275,11 @@ def _list_all_metafiles(
                     **kwargs,
                     **kwargs_update,
                 }
-                metafiles = lister(**lister_kwargs)
-                list_results.append(metafiles)
+                child_metafiles = lister(**lister_kwargs)
+                list_results.append(child_metafiles)
+                next_level_metafiles.extend(child_metafiles.all_items())
+            # Move to the next level for the next iteration
+            current_level_metafiles = next_level_metafiles
     return [
         metafile for list_result in list_results for metafile in list_result.all_items()
     ]
@@ -308,7 +360,7 @@ def put(
     *args,
     **kwargs,
 ) -> Union[Metafile, str]:
-    writer = DeltaCatUrlWriter(url, metafile)
+    writer = DeltaCatUrlWriter(url, metafile=metafile)
     return writer.write(*args, **kwargs)
 
 
@@ -351,6 +403,7 @@ def _copy_external_ray(
         "gz": 35,
         "bz2": 35,
         "zip": 35,
+        "zst": 35,
         "7z": 35,
         "*": 2.5,
     },
@@ -359,7 +412,7 @@ def _copy_external_ray(
     writer_args: Dict[str, Any] = {},
     filesystem: pafs.FileSystem = None,
 ) -> str:
-    print(f"DeltaCAT Copy Invocation Received at: {time.time_ns()}")
+    logger.info(f"DeltaCAT Copy Invocation Received at: {time.time_ns()}")
 
     if not isinstance(src, DeltaCatUrl):
         raise ValueError(f"Expected `src` to be a `DeltaCatUrl` but got `{src}`.")
@@ -367,30 +420,32 @@ def _copy_external_ray(
     #  wait for required resources
     head_cpu_count = int(current_node_resources()["CPU"])
     if minimum_worker_cpus > 0:
-        print(f"Waiting for {minimum_worker_cpus} worker CPUs...")
+        logger.info(f"Waiting for {minimum_worker_cpus} worker CPUs...")
         live_cpu_waiter(
             min_live_cpus=minimum_worker_cpus + head_cpu_count,
         )
-        print(f"{minimum_worker_cpus} worker CPUs found!")
+        logger.info(f"{minimum_worker_cpus} worker CPUs found!")
     # start job execution
     cluster_resources = ray.cluster_resources()
-    print(f"Cluster Resources: {cluster_resources}")
-    print(f"Available Cluster Resources: {ray.available_resources()}")
+    logger.info(f"Cluster Resources: {cluster_resources}")
+    logger.info(f"Available Cluster Resources: {ray.available_resources()}")
     cluster_cpus = int(cluster_resources["CPU"])
-    print(f"Cluster CPUs: {cluster_cpus}")
+    logger.info(f"Cluster CPUs: {cluster_cpus}")
     all_node_resource_keys = live_node_resource_keys()
-    print(f"Found {len(all_node_resource_keys)} live nodes: {all_node_resource_keys}")
+    logger.info(
+        f"Found {len(all_node_resource_keys)} live nodes: {all_node_resource_keys}"
+    )
     worker_node_resource_keys = other_live_node_resource_keys()
-    print(
+    logger.info(
         f"Found {len(worker_node_resource_keys)} live worker nodes: {worker_node_resource_keys}"
     )
     worker_cpu_count = cluster_cpus - head_cpu_count
-    print(f"Total worker CPUs: {worker_cpu_count}")
+    logger.info(f"Total worker CPUs: {worker_cpu_count}")
 
     # estimate memory requirements based on file extension
     estimated_memory_bytes = 0
     if extension_to_memory_multiplier:
-        print(f"Resolving stats collection filesystem for: {src.url_path}.")
+        logger.info(f"Resolving stats collection filesystem for: {src.url_path}.")
         path, filesystem = resolve_path_and_filesystem(src.url_path, filesystem)
         if isinstance(filesystem, pafs.GcsFileSystem):
             from datetime import timedelta
@@ -402,7 +457,7 @@ def _copy_external_ray(
                 anonymous=True,
                 retry_time_limit=timedelta(seconds=10),
             )
-        print(f"Using filesystem {type(filesystem)} to get file size of: {path}")
+        logger.info(f"Using filesystem {type(filesystem)} to get file size of: {path}")
         file_info = get_file_info(path, filesystem)
         if file_info.type != FileType.File:
             raise ValueError(
@@ -413,11 +468,11 @@ def _copy_external_ray(
         if inflation_multiplier is None:
             inflation_multiplier = extension_to_memory_multiplier.get("*")
         estimated_memory_bytes = inflation_multiplier * file_info.size
-        print(
+        logger.info(
             f"Estimated Memory Required for Copy: "
             f"{estimated_memory_bytes/BYTES_PER_GIBIBYTE} GiB"
         )
-    print(f"Starting DeltaCAT Copy at: {time.time_ns()}")
+    logger.info(f"Starting DeltaCAT Copy at: {time.time_ns()}")
 
     index_result = None
     num_cpus = 1
@@ -436,31 +491,31 @@ def _copy_external_ray(
             reader_args=reader_args,
             writer_args=writer_args,
         )
-        print(f"Time to Launch Copy Task: {latency} seconds")
+        logger.info(f"Time to Launch Copy Task: {latency} seconds")
         try:
             index_result, latency = timed_invocation(
                 ray.get,
                 copy_task_pending,
             )
         except OutOfMemoryError as e:
-            print(f"Copy Task Ran Out of Memory: {e}")
+            logger.warning(f"Copy Task Ran Out of Memory: {e}")
             max_single_node_cpus = min(
                 max_allowed_cpus, find_max_single_node_resource_type("CPU")
             )
             num_cpus += 1
             if num_cpus > max_single_node_cpus:
                 raise e
-            print(f"Retrying Failed Copy Task with {num_cpus} dedicated CPUs")
+            logger.info(f"Retrying Failed Copy Task with {num_cpus} dedicated CPUs")
 
-    print(f"Time to Launch Copy Task: {latency} seconds")
-    print(f"Time to Complete Copy Task: {latency} seconds")
+    logger.info(f"Time to Launch Copy Task: {latency} seconds")
+    logger.info(f"Time to Complete Copy Task: {latency} seconds")
 
     total_gib_indexed = index_result.table_size / BYTES_PER_GIBIBYTE
 
-    print(f"Records Copied: {index_result.table_length}")
-    print(f"Bytes Copied: {total_gib_indexed} GiB")
-    print(f"Conversion Rate: {total_gib_indexed/latency} GiB/s")
-    print(f"Finished Copy at: {time.time_ns()}")
+    logger.info(f"Records Copied: {index_result.table_length}")
+    logger.info(f"Bytes Copied: {total_gib_indexed} GiB")
+    logger.info(f"Conversion Rate: {total_gib_indexed/latency} GiB/s")
+    logger.info(f"Finished Copy at: {time.time_ns()}")
 
     return dst.url
 
@@ -484,13 +539,13 @@ def copy_task(
         transforms=transforms,
         reader_args=reader_args,
     )
-    print(f"Time to read {src.url_path}: {latency} seconds")
+    logger.debug(f"Time to read {src.url_path}: {latency} seconds")
 
     table_size = get_table_size(table)
-    print(f"Table Size: {table_size/BYTES_PER_GIBIBYTE} GiB")
+    logger.debug(f"Table Size: {table_size/BYTES_PER_GIBIBYTE} GiB")
 
     table_length = get_table_length(table)
-    print(f"Table Records: {table_length}")
+    logger.debug(f"Table Records: {table_length}")
 
     writer = DeltaCatUrlWriter(dest, dataset_type)
     written_file_path, latency = timed_invocation(
@@ -499,7 +554,7 @@ def copy_task(
         table,
         **writer_args,
     )
-    print(f"Time to write {written_file_path}: {latency}")
+    logger.debug(f"Time to write {written_file_path}: {latency}")
 
     return CopyResult(table_size, table_length)
 

deltacat/aws/constants.py

@@ -1,32 +1,9 @@
-import botocore
 from typing import Set
-from daft.exceptions import DaftTransientError
 from deltacat.utils.common import env_integer, env_string
 
 
 DAFT_MAX_S3_CONNECTIONS_PER_FILE = env_integer("DAFT_MAX_S3_CONNECTIONS_PER_FILE", 8)
-DEFAULT_FILE_READ_TIMEOUT_MS = env_integer(
-    "DEFAULT_FILE_READ_TIMEOUT_MS", 300_000
-)  # 5 mins
 BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 5)
 BOTO_TIMEOUT_ERROR_CODES: Set[str] = {"ReadTimeoutError", "ConnectTimeoutError"}
 BOTO_THROTTLING_ERROR_CODES: Set[str] = {"Throttling", "SlowDown"}
-RETRYABLE_TRANSIENT_ERRORS = (
-    OSError,
-    botocore.exceptions.ConnectionError,
-    botocore.exceptions.HTTPClientError,
-    botocore.exceptions.NoCredentialsError,
-    botocore.exceptions.ConnectTimeoutError,
-    botocore.exceptions.ReadTimeoutError,
-    DaftTransientError,
-)
 AWS_REGION = env_string("AWS_REGION", "us-east-1")
-UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY = env_integer(
-    "UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY", 10 * 60
-)
-UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY = env_integer(
-    "UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY", 30 * 60
-)
-DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY = env_integer(
-    "DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY", 30 * 60
-)