deltacat 1.1.35__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/examples/iceberg/iceberg_bucket_writer.py

@@ -0,0 +1,139 @@
+import os
+import logging
+
+import daft
+import deltacat as dc
+
+from deltacat import logs
+from deltacat import IcebergCatalog
+from deltacat.examples.common.fixtures import (
+    store_cli_args_in_os_environ,
+)
+
+from pyiceberg.schema import (
+    Schema,
+    NestedField,
+    DoubleType,
+    StringType,
+)
+from pyiceberg.partitioning import PartitionSpec, PartitionField
+from pyiceberg.transforms import BucketTransform
+
+from deltacat.storage.iceberg.model import (
+    SchemaMapper,
+    PartitionSchemeMapper,
+)
+from deltacat.env import create_ray_runtime_environment
+
+# initialize the driver logger
+driver_logger = logs.configure_application_logger(logging.getLogger(__name__))
+
+
+def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
+    # create any runtime environment required to run the example
+    runtime_env = create_ray_runtime_environment()
+
+    # Start by initializing DeltaCAT and registering available Catalogs.
+    # Ray will be initialized automatically via `ray.init()`.
+    # Only the `iceberg` data catalog is provided so it will become the default.
+    # If initializing multiple catalogs, use the `default_catalog_name` param
+    # to specify which catalog should be the default.
+    dc.init(
+        catalogs={
+            # the name of the DeltaCAT catalog is "iceberg"
+            "iceberg": dc.Catalog(
+                # Apache Iceberg implementation of deltacat.catalog.interface
+                impl=IcebergCatalog,
+                # kwargs for pyiceberg.catalog.load_catalog start here...
+                # the name of the Iceberg catalog is "example-iceberg-catalog"
+                name="example-iceberg-catalog",
+                # for additional properties see:
+                # https://py.iceberg.apache.org/configuration/
+                properties={
+                    "type": "glue",
+                    "region_name": "us-east-1",
+                    "warehouse": warehouse,
+                },
+            )
+        },
+        # pass the runtime environment into ray.init()
+        ray_init_args={"runtime_env": runtime_env},
+    )
+
+    # define a native Iceberg table schema
+    schema = Schema(
+        NestedField(field_id=1, name="symbol", field_type=StringType(), required=True),
+        NestedField(field_id=2, name="bid", field_type=DoubleType(), required=False),
+        NestedField(field_id=3, name="ask", field_type=DoubleType(), required=False),
+    )
+
+    # define a native Iceberg partition spec
+    partition_spec = PartitionSpec(
+        PartitionField(
+            source_id=1,
+            field_id=1000,
+            transform=BucketTransform(2),
+            name="symbol_bucket",
+        )
+    )
+
+    # define a native Iceberg sort order
+    # sort_order = SortOrder(SortField(source_id=1, transform=IdentityTransform()))
+
+    # define the Daft dataframe to write
+    df = daft.from_pydict(
+        {
+            "symbol": ["amzn", "goog", "meta", "msft"],
+            "bid": [157.16, 150.55, 392.03, 403.25],
+            "ask": [157.17, 150.56, 392.09, 403.27],
+        }
+    )
+
+    # write to a table named `test_namespace.test_table_bucketed`
+    # we don't need to specify which catalog to create this table in since
+    # only the "iceberg" catalog is available
+    table_name = "test_table_bucketed"
+    namespace = "test_namespace"
+    print(f"Creating Glue Table: {namespace}.{table_name}")
+    dc.write_to_table(
+        data=df,
+        # path=warehouse + "/datafiles",
+        table=table_name,
+        namespace=namespace,
+        schema=SchemaMapper.map(schema),
+        partition_scheme=PartitionSchemeMapper.map(partition_spec, schema),
+        # sort_keys=SortSchemeMapper.map(sort_order, schema),
+    )
+
+    print(f"Getting Glue Table: {namespace}.{table_name}")
+    table_definition = dc.get_table(table_name, namespace)
+    print(f"Retrieved Glue Table: {table_definition}")
+
+
+if __name__ == "__main__":
+    example_script_args = [
+        (
+            [
+                "--warehouse",
+            ],
+            {
+                "help": "S3 path for Iceberg file storage.",
+                "type": str,
+            },
+        ),
+        (
+            [
+                "--STAGE",
+            ],
+            {
+                "help": "Example runtime environment stage (e.g. dev, alpha, beta, prod).",
+                "type": str,
+            },
+        ),
+    ]
+
+    # store any CLI args in the runtime environment
+    store_cli_args_in_os_environ(example_script_args)
+
+    # run the example using os.environ as kwargs
+    run(**os.environ)

deltacat/examples/iceberg/iceberg_reader.py

@@ -0,0 +1,149 @@
+import os
+import logging
+import deltacat as dc
+
+from deltacat import logs
+from deltacat import IcebergCatalog
+from deltacat.examples.common.fixtures import (
+    store_cli_args_in_os_environ,
+)
+
+from pyiceberg.schema import (
+    Schema,
+    NestedField,
+    DoubleType,
+    StringType,
+    TimestampType,
+    FloatType,
+    StructType,
+)
+from pyiceberg.partitioning import PartitionSpec, PartitionField
+from pyiceberg.transforms import DayTransform, IdentityTransform
+from pyiceberg.table.sorting import SortField, SortOrder
+
+from deltacat.exceptions import TableAlreadyExistsError
+from deltacat.storage.iceberg.model import (
+    SchemaMapper,
+    PartitionSchemeMapper,
+    SortSchemeMapper,
+)
+from deltacat.env import create_ray_runtime_environment
+
+# initialize the driver logger
+driver_logger = logs.configure_application_logger(logging.getLogger(__name__))
+
+
+def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
+    # create any runtime environment required to run the example
+    runtime_env = create_ray_runtime_environment()
+
+    # Start by initializing DeltaCAT and registering available Catalogs.
+    # Ray will be initialized automatically via `ray.init()`.
+    # Only the `iceberg` data catalog is provided so it will become the default.
+    # If initializing multiple catalogs, use the `default_catalog_name` param
+    # to specify which catalog should be the default.
+    dc.init(
+        catalogs={
+            # the name of the DeltaCAT catalog is "iceberg"
+            "iceberg": dc.Catalog(
+                # Apache Iceberg implementation of deltacat.catalog.interface
+                impl=IcebergCatalog,
+                # kwargs for pyiceberg.catalog.load_catalog start here...
+                # the name of the Iceberg catalog is "example-iceberg-catalog"
+                name="example-iceberg-catalog",
+                # for additional properties see:
+                # https://py.iceberg.apache.org/configuration/
+                properties={
+                    "type": "glue",
+                    "region_name": "us-east-1",
+                    "warehouse": warehouse,
+                },
+            )
+        },
+        # pass the runtime environment into ray.init()
+        ray_init_args={"runtime_env": runtime_env},
+    )
+
+    # define a native Iceberg table schema
+    schema = Schema(
+        NestedField(
+            field_id=1, name="datetime", field_type=TimestampType(), required=True
+        ),
+        NestedField(field_id=2, name="symbol", field_type=StringType(), required=True),
+        NestedField(field_id=3, name="bid", field_type=FloatType(), required=False),
+        NestedField(field_id=4, name="ask", field_type=DoubleType(), required=False),
+        NestedField(
+            field_id=5,
+            name="details",
+            field_type=StructType(
+                NestedField(
+                    field_id=6,
+                    name="created_by",
+                    field_type=StringType(),
+                    required=False,
+                ),
+            ),
+            required=False,
+        ),
+    )
+
+    # define a native Iceberg partition spec
+    partition_spec = PartitionSpec(
+        PartitionField(
+            source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day"
+        )
+    )
+
+    # define a native Iceberg sort order
+    sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform()))
+
+    # create a table named `test_namespace.test_table`
+    # we don't need to specify which catalog to create this table in since
+    # only the "iceberg" catalog is available
+    table_name = "test_table"
+    namespace = "test_namespace"
+    print(f"Creating Glue Table: {namespace}.{table_name}")
+    try:
+        table_definition = dc.create_table(
+            table=table_name,
+            namespace=namespace,
+            schema=SchemaMapper.map(schema),
+            partition_scheme=PartitionSchemeMapper.map(partition_spec, schema),
+            sort_keys=SortSchemeMapper.map(sort_order, schema),
+        )
+        print(f"Created Glue Table: {table_definition}")
+    except TableAlreadyExistsError:
+        print(f"Glue Table `{namespace}.{table_name}` already exists.")
+
+    print(f"Getting Glue Table: {namespace}.{table_name}")
+    table_definition = dc.get_table(table_name, namespace)
+    print(f"Retrieved Glue Table: {table_definition}")
+
+
+if __name__ == "__main__":
+    example_script_args = [
+        (
+            [
+                "--warehouse",
+            ],
+            {
+                "help": "S3 path for Iceberg file storage.",
+                "type": str,
+            },
+        ),
+        (
+            [
+                "--STAGE",
+            ],
+            {
+                "help": "Example runtime environment stage (e.g. dev, alpha, beta, prod).",
+                "type": str,
+            },
+        ),
+    ]
+
+    # store any CLI args in the runtime environment
+    store_cli_args_in_os_environ(example_script_args)
+
+    # run the example using os.environ as kwargs
+    run(**os.environ)

deltacat/exceptions.py

@@ -1,10 +1,16 @@
 from __future__ import annotations
 from enum import Enum
-import botocore
-import ray
+from typing import Callable
 import logging
+
 import tenacity
-from deltacat import logs
+
+from pyarrow.lib import ArrowException, ArrowInvalid, ArrowCapacityError
+
+import botocore
+from botocore.exceptions import BotoCoreError
+
+import ray
 from ray.exceptions import (
     RayError,
     RayTaskError,
@@ -13,14 +19,14 @@ from ray.exceptions import (
     NodeDiedError,
     OutOfMemoryError,
 )
-from deltacat.storage import interface as DeltaCatStorage
-from pyarrow.lib import ArrowException, ArrowInvalid, ArrowCapacityError
-from botocore.exceptions import BotoCoreError
-from typing import Callable
+
+from daft.exceptions import DaftTransientError, DaftCoreException
+
+import deltacat as dc
+from deltacat import logs
 from deltacat.utils.ray_utils.runtime import (
     get_current_ray_task_id,
 )
-from daft.exceptions import DaftTransientError, DaftCoreException
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -64,6 +70,14 @@ class DeltaCatErrorNames(str, Enum):
     UNCLASSIFIED_DELTACAT_ERROR = "UnclassifiedDeltaCatError"
     UNRECOGNIZED_RAY_TASK_ERROR = "UnrecognizedRayTaskError"
 
+    NAMESPACE_NOT_FOUND_ERROR = "NamespaceNotFoundError"
+    TABLE_NOT_FOUND_ERROR = "TableNotFoundError"
+    TABLE_VERSION_NOT_FOUND_ERROR = "TableVersionNotFoundError"
+    STREAM_NOT_FOUND_ERROR = "StreamNotFoundError"
+    DELTA_NOT_FOUND_ERROR = "DeltaNotFoundError"
+    TABLE_ALREADY_EXISTS_ERROR = "TableAlreadyExistsError"
+    NAMESPACE_ALREADY_EXISTS_ERROR = "NamespaceAlreadyExistsError"
+
 
 class DeltaCatError(Exception):
     def __init__(self, *args, **kwargs):
@@ -206,6 +220,34 @@ class UnrecognizedRayTaskError(NonRetryableError):
     error_name = DeltaCatErrorNames.UNRECOGNIZED_RAY_TASK_ERROR.value
 
 
+class NamespaceNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.NAMESPACE_NOT_FOUND_ERROR.value
+
+
+class TableNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.TABLE_NOT_FOUND_ERROR.value
+
+
+class TableVersionNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.TABLE_VERSION_NOT_FOUND_ERROR.value
+
+
+class StreamNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.STREAM_NOT_FOUND_ERROR.value
+
+
+class DeltaNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.DELTA_NOT_FOUND_ERROR.value
+
+
+class TableAlreadyExistsError(NonRetryableError):
+    error_name = DeltaCatErrorNames.TABLE_ALREADY_EXISTS_ERROR.value
+
+
+class NamespaceAlreadyExistsError(NonRetryableError):
+    error_name = DeltaCatErrorNames.TABLE_ALREADY_EXISTS_ERROR.value
+
+
 def categorize_errors(func: Callable):
     def wrapper(*args, **kwargs):
         try:
@@ -238,7 +280,7 @@ def categorize_errors(func: Callable):
 
 def categorize_deltacat_exception(
     e: BaseException,
-    deltacat_storage: DeltaCatStorage = None,
+    deltacat_storage: dc.storage.interface = None,
     deltacat_storage_kwargs: dict = None,
 ):
     if deltacat_storage_kwargs is None:

deltacat/logs.py

@@ -18,6 +18,7 @@ from deltacat.constants import (
     DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
     DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
     DELTACAT_LOGGER_CONTEXT,
+    DELTACAT_LOGGER_USE_SINGLE_HANDLER,
 )
 
 DEFAULT_LOG_LEVEL = "INFO"
@@ -226,6 +227,7 @@ def _configure_logger(
     # This maintains log level of rotating file handlers
     primary_log_level = log_level
     logger.propagate = False
+    needs_handler = True
     if log_level <= logging.getLevelName("DEBUG"):
         if not _file_handler_exists(logger, log_dir, debug_log_base_file_name):
             handler = _create_rotating_file_handler(
@@ -235,8 +237,9 @@ def _configure_logger(
                 context_kwargs=context_kwargs,
             )
             _add_logger_handler(logger, handler)
+            needs_handler = not DELTACAT_LOGGER_USE_SINGLE_HANDLER
             primary_log_level = logging.getLevelName("INFO")
-    if not _file_handler_exists(logger, log_dir, log_base_file_name):
+    if not _file_handler_exists(logger, log_dir, log_base_file_name) and needs_handler:
         handler = _create_rotating_file_handler(
             log_dir,
             log_base_file_name,

deltacat/storage/__init__.py

@@ -1,24 +1,63 @@
-from deltacat.aws.redshift import (
+from deltacat.storage.model.manifest import (
+    EntryType,
+    EntryParams,
     Manifest,
     ManifestAuthor,
     ManifestEntry,
     ManifestEntryList,
     ManifestMeta,
 )
-from deltacat.storage.model.delta import Delta, DeltaLocator
+from deltacat.storage.model.delta import (
+    Delta,
+    DeltaLocator,
+    DeltaProperties,
+)
 from deltacat.storage.model.list_result import ListResult
 from deltacat.storage.model.locator import Locator
-from deltacat.storage.model.namespace import Namespace, NamespaceLocator
-from deltacat.storage.model.partition import Partition, PartitionLocator
-from deltacat.storage.model.stream import Stream, StreamLocator
-from deltacat.storage.model.table import Table, TableLocator
-from deltacat.storage.model.table_version import TableVersion, TableVersionLocator
-from deltacat.storage.model.delete_parameters import DeleteParameters
-from deltacat.storage.model.partition_spec import (
-    PartitionFilter,
+from deltacat.storage.model.metafile import (
+    Metafile,
+)
+from deltacat.storage.model.transaction import (
+    TransactionOperation,
+    Transaction,
+)
+from deltacat.storage.model.namespace import (
+    Namespace,
+    NamespaceLocator,
+    NamespaceProperties,
+)
+from deltacat.storage.model.partition import (
+    Partition,
+    PartitionLocator,
+    PartitionLocatorAlias,
+    PartitionKey,
+    PartitionScheme,
+    PartitionSchemeList,
     PartitionValues,
-    DeltaPartitionSpec,
-    StreamPartitionSpec,
+)
+from deltacat.storage.model.schema import (
+    Field,
+    FieldId,
+    FieldLocator,
+    FieldName,
+    NestedFieldName,
+    Schema,
+    SchemaList,
+)
+from deltacat.storage.model.stream import (
+    Stream,
+    StreamLocator,
+    StreamLocatorAlias,
+)
+from deltacat.storage.model.table import (
+    Table,
+    TableLocator,
+    TableProperties,
+)
+from deltacat.storage.model.table_version import (
+    TableVersion,
+    TableVersionLocator,
+    TableVersionProperties,
 )
 from deltacat.storage.model.transform import (
     Transform,
@@ -26,9 +65,17 @@ from deltacat.storage.model.transform import (
     TransformParameters,
     BucketingStrategy,
     BucketTransformParameters,
-    IdentityTransformParameters,
+    TruncateTransformParameters,
+    BucketTransform,
+    IdentityTransform,
+    VoidTransform,
+    UnknownTransform,
+    HourTransform,
+    DayTransform,
+    MonthTransform,
+    YearTransform,
+    TruncateTransform,
 )
-
 from deltacat.storage.model.types import (
     CommitState,
     DeltaType,
@@ -36,18 +83,39 @@ from deltacat.storage.model.types import (
     LifecycleState,
     LocalDataset,
     LocalTable,
+    NullOrder,
     SchemaConsistencyType,
+    StreamFormat,
+    SortOrder,
+    TransactionType,
+    TransactionOperationType,
+)
+from deltacat.storage.model.sort_key import (
+    SortKey,
+    SortScheme,
+    SortSchemeList,
 )
-from deltacat.storage.model.sort_key import SortKey, SortOrder
+from deltacat.storage.main import impl as metastore
 
 __all__ = [
+    "BucketingStrategy",
+    "BucketTransform",
+    "BucketTransformParameters",
     "CommitState",
+    "DayTransform",
     "Delta",
     "DeltaLocator",
-    "Partition",
-    "DeleteParameters",
+    "DeltaProperties",
     "DeltaType",
     "DistributedDataset",
+    "EntryType",
+    "EntryParams",
+    "Field",
+    "FieldId",
+    "FieldLocator",
+    "FieldName",
+    "HourTransform",
+    "IdentityTransform",
     "LifecycleState",
     "ListResult",
     "LocalDataset",
@@ -56,28 +124,50 @@ __all__ = [
     "Manifest",
     "ManifestAuthor",
     "ManifestEntry",
-    "ManifestMeta",
     "ManifestEntryList",
+    "ManifestMeta",
+    "Metafile",
+    "metastore",
+    "MonthTransform",
     "Namespace",
     "NamespaceLocator",
+    "NamespaceProperties",
+    "NestedFieldName",
+    "NullOrder",
+    "Partition",
+    "PartitionKey",
     "PartitionLocator",
-    "Stream",
+    "PartitionLocatorAlias",
+    "PartitionScheme",
+    "PartitionSchemeList",
+    "PartitionValues",
+    "Schema",
+    "SchemaList",
     "SchemaConsistencyType",
+    "SortKey",
+    "SortOrder",
+    "SortScheme",
+    "SortSchemeList",
+    "Stream",
+    "StreamFormat",
     "StreamLocator",
+    "StreamLocatorAlias",
     "Table",
     "TableLocator",
+    "TableProperties",
     "TableVersion",
     "TableVersionLocator",
-    "SortKey",
-    "SortOrder",
-    "PartitionFilter",
-    "PartitionValues",
-    "DeltaPartitionSpec",
-    "StreamPartitionSpec",
+    "TableVersionProperties",
+    "Transaction",
+    "TransactionOperation",
+    "TransactionOperationType",
+    "TransactionType",
     "Transform",
     "TransformName",
     "TransformParameters",
-    "BucketingStrategy",
-    "BucketTransformParameters",
-    "IdentityTransformParameters",
+    "TruncateTransform",
+    "TruncateTransformParameters",
+    "UnknownTransform",
+    "VoidTransform",
+    "YearTransform",
 ]

deltacat/storage/iceberg/iceberg_scan_planner.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+from pyiceberg.catalog import Catalog
+from deltacat.storage.model.scan.push_down import Pushdown
+from deltacat.storage.model.scan.scan_plan import ScanPlan
+from deltacat.storage.model.scan.scan_task import FileScanTask, DataFile
+from deltacat.storage.util.scan_planner import ScanPlanner
+from deltacat.storage.iceberg.impl import _try_load_iceberg_table
+
+
+class IcebergScanPlanner(ScanPlanner):
+    def __init__(self, catalog: Catalog):
+        self.catalog = catalog
+
+    def create_scan_plan(
+        self,
+        table_name: str,
+        namespace: Optional[str] = None,
+        pushdown: Optional[Pushdown] = None,
+    ) -> ScanPlan:
+        iceberg_table = _try_load_iceberg_table(
+            self.catalog, namespace=namespace, table_name=table_name
+        )
+        file_scan_tasks = []
+        # TODO: implement predicate pushdown to Iceberg
+        for scan_task in iceberg_table.scan().plan_files():
+            file_scan_tasks.append(FileScanTask([DataFile(scan_task.file.file_path)]))
+        return ScanPlan(file_scan_tasks)