deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/examples/experimental/iceberg/converter/beam/utils/spark.py

@@ -0,0 +1,263 @@
+"""
+Spark SQL utilities for Iceberg table operations.
+
+This module provides Beam DoFn classes that use Spark SQL to work with Iceberg tables,
+"""
+
+import os
+import apache_beam as beam
+from apache_beam import Row
+
+
+class SparkSQLIcebergRead(beam.DoFn):
+    """
+    Custom Beam DoFn that uses Spark SQL to read Iceberg tables.
+    """
+
+    def __init__(
+        self,
+        table_name: str,
+        catalog_uri: str = "http://localhost:8181",
+        warehouse: str = "warehouse/",
+    ):
+        """
+        Initialize the Spark SQL reader.
+
+        Args:
+            table_name: Name of the Iceberg table
+            catalog_uri: URI of the Iceberg REST catalog
+            warehouse: Warehouse path
+        """
+        self.table_name = table_name
+        self.catalog_uri = catalog_uri
+        self.warehouse = warehouse
+        self.spark = None
+
+    def setup(self):
+        """Set up Spark session (called once per worker)."""
+        try:
+            from pyspark.sql import SparkSession
+            import importlib.metadata
+
+            # Get Spark version for dependency resolution
+            try:
+                spark_version = ".".join(
+                    importlib.metadata.version("pyspark").split(".")[:2]
+                )
+            except Exception:
+                spark_version = "3.5"  # Default fallback
+
+            scala_version = "2.12"
+            iceberg_version = "1.6.0"
+
+            print(f"🔧 Setting up Spark session for reading {self.table_name}")
+            print(f"   - Spark version: {spark_version}")
+            print(f"   - Iceberg version: {iceberg_version}")
+
+            # Set Spark packages for Iceberg runtime
+            os.environ["PYSPARK_SUBMIT_ARGS"] = (
+                f"--packages org.apache.iceberg:iceberg-spark-runtime-{spark_version}_{scala_version}:{iceberg_version} "
+                f"pyspark-shell"
+            )
+
+            # Create Spark session with Iceberg REST catalog configuration
+            self.spark = (
+                SparkSession.builder.appName(f"DeltaCAT Read - {self.table_name}")
+                .config("spark.sql.session.timeZone", "UTC")
+                .config(
+                    "spark.serializer", "org.apache.spark.serializer.KryoSerializer"
+                )
+                .config(
+                    "spark.sql.extensions",
+                    "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+                )
+                # Configure REST catalog
+                .config(
+                    "spark.sql.catalog.rest", "org.apache.iceberg.spark.SparkCatalog"
+                )
+                .config("spark.sql.catalog.rest.type", "rest")
+                .config("spark.sql.catalog.rest.uri", self.catalog_uri)
+                .config("spark.sql.catalog.rest.warehouse", self.warehouse)
+                # Set REST as default catalog
+                .config("spark.sql.defaultCatalog", "rest")
+                # Local mode configuration (within Beam workers)
+                .config("spark.master", "local[1]")  # Single thread per worker
+                .config("spark.sql.adaptive.enabled", "true")
+                # Networking binding
+                .config("spark.driver.bindAddress", "127.0.0.1")
+                .config("spark.driver.host", "127.0.0.1")
+                .config("spark.ui.enabled", "false")
+                .config("spark.sql.adaptive.coalescePartitions.enabled", "false")
+                .getOrCreate()
+            )
+
+            print(f"✅ Spark session created successfully")
+
+        except Exception as e:
+            print(f"❌ Failed to set up Spark session: {e}")
+            raise
+
+    def teardown(self):
+        """Clean up Spark session (called once per worker)."""
+        if self.spark:
+            try:
+                self.spark.stop()
+                print("✅ Spark session stopped")
+            except Exception as e:
+                print(f"⚠️ Error stopping Spark session: {e}")
+
+    def process(self, element):
+        """
+        Process element (read from Iceberg table using Spark SQL).
+
+        Args:
+            element: Input element (not used, just triggers the read)
+
+        Yields:
+            Records from the Iceberg table
+        """
+        try:
+            if not self.spark:
+                raise RuntimeError("Spark session not initialized")
+
+            print(f"📖 Reading table {self.table_name} using Spark SQL")
+
+            # Read from Iceberg table using Spark SQL
+            df = self.spark.sql(f"SELECT * FROM {self.table_name}")
+
+            # Collect all records
+            records = df.collect()
+
+            print(f"📊 Successfully read {len(records)} records from {self.table_name}")
+
+            # Convert Spark rows to Beam Row objects and yield
+            for row in records:
+                row_dict = row.asDict()
+                # Convert to Beam Row for consistency with write mode
+                beam_row = Row(**row_dict)
+                yield beam_row
+
+        except Exception as e:
+            print(f"❌ Failed to read from table {self.table_name}: {e}")
+            raise
+
+
+class SparkSQLIcebergRewrite(beam.DoFn):
+    """
+    Custom Beam DoFn that uses Spark SQL to rewrite Iceberg table data files.
+
+    This uses Spark's rewrite_data_files procedure to materialize positional deletes
+    by rewriting data files. The result is a "clean" table without positional deletes.
+    """
+
+    def __init__(self, catalog_uri, warehouse_path, table_name):
+        self.catalog_uri = catalog_uri
+        self.warehouse_path = warehouse_path
+        self.table_name = table_name
+
+    def setup(self):
+        """Initialize Spark session for rewrite operations."""
+        try:
+            from pyspark.sql import SparkSession
+            import importlib.metadata
+
+            print(f"🔧 Setting up Spark session for rewriting {self.table_name}")
+
+            # Detect Spark version for appropriate Iceberg runtime
+            spark_version = importlib.metadata.version("pyspark")
+            major_minor = ".".join(spark_version.split(".")[:2])
+            print(f"   - Spark version: {major_minor}")
+            print(f"   - Iceberg version: 1.6.0")
+
+            # Configure Spark with Iceberg
+            self.spark = (
+                SparkSession.builder.appName("IcebergRewrite")
+                .config(
+                    "spark.jars.packages",
+                    f"org.apache.iceberg:iceberg-spark-runtime-{major_minor}_2.12:1.6.0",
+                )
+                .config(
+                    "spark.sql.extensions",
+                    "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+                )
+                .config(
+                    "spark.sql.catalog.spark_catalog",
+                    "org.apache.iceberg.spark.SparkSessionCatalog",
+                )
+                .config("spark.sql.catalog.spark_catalog.type", "rest")
+                .config("spark.sql.catalog.spark_catalog.uri", self.catalog_uri)
+                .config(
+                    "spark.sql.catalog.spark_catalog.warehouse", self.warehouse_path
+                )
+                .config("spark.driver.bindAddress", "127.0.0.1")
+                .config("spark.driver.host", "127.0.0.1")
+                .config("spark.ui.enabled", "false")
+                .getOrCreate()
+            )
+
+            print("✅ Spark session created successfully")
+
+        except ImportError as e:
+            raise RuntimeError(
+                f"PySpark is required for rewrite mode. Install with: pip install pyspark"
+            ) from e
+        except Exception as e:
+            raise RuntimeError(f"Failed to create Spark session: {e}") from e
+
+    def process(self, element):
+        """Rewrite table data files to materialize positional deletes."""
+        try:
+            print(
+                f"📋 Rewriting table {self.table_name} to materialize positional deletes"
+            )
+
+            # Use Spark's rewrite_data_files procedure with delete_file_threshold=1
+            # This forces rewrite even when there's only 1 positional delete file
+            rewrite_sql = f"""
+            CALL spark_catalog.system.rewrite_data_files(
+                table => '{self.table_name}',
+                options => map('delete-file-threshold', '1')
+            )
+            """
+
+            print(f"🔄 Executing rewrite procedure with delete_file_threshold=1...")
+            print(f"   SQL: {rewrite_sql.strip()}")
+            print(
+                f"   Rationale: Forces rewrite even with single positional delete file"
+            )
+
+            result = self.spark.sql(rewrite_sql)
+
+            # Collect results to see what was rewritten
+            rewrite_result = result.collect()[0]
+            print(f"📊 Rewrite result: {rewrite_result}")
+
+            # Check if we actually rewrote anything
+            if rewrite_result.rewritten_data_files_count > 0:
+                print(
+                    f"✅ Successfully rewrote {rewrite_result.rewritten_data_files_count} data files"
+                )
+                print(
+                    f"   - Added {rewrite_result.added_data_files_count} new data files"
+                )
+                print(f"   - Rewrote {rewrite_result.rewritten_bytes_count} bytes")
+                print(f"   - Positional deletes have been materialized!")
+            else:
+                print(f"⚠️  No files were rewritten (rewritten_data_files_count=0)")
+                print(f"   - This may indicate no positional deletes exist")
+                print(f"   - Or the table may already be in optimal state")
+
+            yield f"Rewrite completed for {self.table_name}"
+
+        except Exception as e:
+            print(f"❌ Error during rewrite: {e}")
+            import traceback
+
+            traceback.print_exc()
+            yield f"Rewrite failed for {self.table_name}: {e}"
+
+    def teardown(self):
+        """Clean up Spark session."""
+        if hasattr(self, "spark"):
+            print("✅ Spark session stopped")
+            self.spark.stop()

deltacat/examples/{iceberg → experimental/iceberg}/iceberg_bucket_writer.py

@@ -9,10 +9,8 @@ import deltacat as dc
 
 from deltacat import logs
 from deltacat import IcebergCatalog
-from deltacat.catalog.iceberg import IcebergCatalogConfig
-from deltacat.examples.common.fixtures import (
-    store_cli_args_in_os_environ,
-)
+from deltacat.experimental.catalog.iceberg import IcebergCatalogConfig
+from env import store_cli_args_in_os_environ
 
 from pyiceberg.schema import (
     Schema,
@@ -23,7 +21,7 @@ from pyiceberg.schema import (
 from pyiceberg.partitioning import PartitionSpec, PartitionField
 from pyiceberg.transforms import BucketTransform
 
-from deltacat.storage.iceberg.model import (
+from deltacat.experimental.storage.iceberg.model import (
     SchemaMapper,
     PartitionSchemeMapper,
 )

deltacat/examples/{iceberg → experimental/iceberg}/iceberg_reader.py

@@ -4,9 +4,7 @@ import deltacat as dc
 
 from deltacat import logs
 from deltacat import IcebergCatalog
-from deltacat.examples.common.fixtures import (
-    store_cli_args_in_os_environ,
-)
+from env import store_cli_args_in_os_environ
 
 from pyiceberg.schema import (
     Schema,
@@ -22,7 +20,7 @@ from pyiceberg.transforms import DayTransform, IdentityTransform
 from pyiceberg.table.sorting import SortField, SortOrder
 
 from deltacat.exceptions import TableAlreadyExistsError
-from deltacat.storage.iceberg.model import (
+from deltacat.experimental.storage.iceberg.model import (
     SchemaMapper,
     PartitionSchemeMapper,
     SortSchemeMapper,

deltacat/examples/indexer/indexer.py

@@ -59,8 +59,8 @@ def run(
             "use_pyarrow": True,  # use the native pyarrow reader
         },
         # writer arguments to pass to the default writer (polars)
-        # for the given parquet-based datasink, it accepts the same
-        # arguments as polars.DataFrame.write_parquet except for `file`
+        # for the given parquet-based datasink, it generally accepts the same
+        # arguments as polars.DataFrame.write_{dest-type} except for `file`
         writer_args={
             "compression": "lz4",  # faster compression & decompression
             # "compression": "zstd",  # better compression ratio

deltacat/examples/indexer/job_runner.py

@@ -64,8 +64,7 @@ def run_sync(
     cluster_cfg_file_path = working_dir.joinpath(cloud).joinpath("deltacat.yaml")
     client = job_client(cluster_cfg_file_path, restart_ray=restart_ray)
     job_number = 0
-    while jobs_to_submit > 0:
-        jobs_to_submit -= 1
+    while job_number < jobs_to_submit:
         job_dest = dest + f".{job_number}"
         job_run_result = client.run_job(
             # Entrypoint shell command to execute

deltacat/exceptions.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 from enum import Enum
-from typing import Callable
+from typing import Callable, Optional, TYPE_CHECKING
 import logging
 
 import tenacity
@@ -28,6 +28,9 @@ from deltacat.utils.ray_utils.runtime import (
     get_current_ray_task_id,
 )
 
+if TYPE_CHECKING:
+    from deltacat.storage.model.schema import FieldLocator
+
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 DELTACAT_STORAGE_PARAM = "deltacat_storage"
@@ -74,9 +77,18 @@ class DeltaCatErrorNames(str, Enum):
     TABLE_NOT_FOUND_ERROR = "TableNotFoundError"
     TABLE_VERSION_NOT_FOUND_ERROR = "TableVersionNotFoundError"
     STREAM_NOT_FOUND_ERROR = "StreamNotFoundError"
+    PARTITION_NOT_FOUND_ERROR = "PartitionNotFoundError"
     DELTA_NOT_FOUND_ERROR = "DeltaNotFoundError"
     TABLE_ALREADY_EXISTS_ERROR = "TableAlreadyExistsError"
+    TABLE_VERSION_ALREADY_EXISTS_ERROR = "TableVersionAlreadyExistsError"
     NAMESPACE_ALREADY_EXISTS_ERROR = "NamespaceAlreadyExistsError"
+    SCHEMA_COMPATIBILITY_ERROR = "SchemaCompatibilityError"
+    SCHEMA_VALIDATION_ERROR = "SchemaValidationError"
+    TABLE_VALIDATION_ERROR = "TableValidationError"
+    CONCURRENT_MODIFICATION_ERROR = "ConcurrentModificationError"
+    OBJECT_NOT_FOUND_ERROR = "ObjectNotFoundError"
+    OBJECT_DELETED_ERROR = "ObjectDeletedError"
+    OBJECT_ALREADY_EXISTS_ERROR = "ObjectAlreadyExistsError"
 
 
 class DeltaCatError(Exception):
@@ -87,9 +99,12 @@ class DeltaCatError(Exception):
         super().__init__(*args, **kwargs)
 
     def _get_ray_task_id_and_node_ip(self):
-        task_id = get_current_ray_task_id()
-        node_ip = ray.util.get_node_ip_address()
-        return task_id, node_ip
+        if ray.is_initialized():
+            task_id = get_current_ray_task_id()
+            node_ip = ray.util.get_node_ip_address()
+            return task_id, node_ip
+        else:
+            return None, None
 
 
 class NonRetryableError(DeltaCatError):
@@ -232,6 +247,10 @@ class TableVersionNotFoundError(NonRetryableError):
     error_name = DeltaCatErrorNames.TABLE_VERSION_NOT_FOUND_ERROR.value
 
 
+class PartitionNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.PARTITION_NOT_FOUND_ERROR.value
+
+
 class StreamNotFoundError(NonRetryableError):
     error_name = DeltaCatErrorNames.STREAM_NOT_FOUND_ERROR.value
 
@@ -244,10 +263,53 @@ class TableAlreadyExistsError(NonRetryableError):
     error_name = DeltaCatErrorNames.TABLE_ALREADY_EXISTS_ERROR.value
 
 
+class TableVersionAlreadyExistsError(NonRetryableError):
+    error_name = DeltaCatErrorNames.TABLE_VERSION_ALREADY_EXISTS_ERROR.value
+
+
 class NamespaceAlreadyExistsError(NonRetryableError):
     error_name = DeltaCatErrorNames.TABLE_ALREADY_EXISTS_ERROR.value
 
 
+class ObjectNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.OBJECT_NOT_FOUND_ERROR.value
+
+
+class ObjectDeletedError(NonRetryableError):
+    error_name = DeltaCatErrorNames.OBJECT_DELETED_ERROR.value
+
+
+class ObjectAlreadyExistsError(NonRetryableError):
+    error_name = DeltaCatErrorNames.OBJECT_ALREADY_EXISTS_ERROR.value
+
+
+class ConcurrentModificationError(NonRetryableError):
+    error_name = DeltaCatErrorNames.CONCURRENT_MODIFICATION_ERROR.value
+
+
+class SchemaValidationError(NonRetryableError):
+    error_name = DeltaCatErrorNames.SCHEMA_VALIDATION_ERROR.value
+
+
+class TableValidationError(NonRetryableError):
+    error_name = DeltaCatErrorNames.TABLE_VALIDATION_ERROR.value
+
+
+class SchemaCompatibilityError(NonRetryableError):
+    error_name = DeltaCatErrorNames.SCHEMA_COMPATIBILITY_ERROR.value
+    """Raised when a schema update would break backward compatibility."""
+
+    def __init__(
+        self,
+        message: str,
+        field_locator: Optional[FieldLocator] = None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(message, *args, **kwargs)
+        self.field_locator = field_locator
+
+
 def categorize_errors(func: Callable):
     def wrapper(*args, **kwargs):
         try:

deltacat/experimental/catalog/iceberg/__init__.py

@@ -0,0 +1,6 @@
+from deltacat.experimental.catalog.iceberg.iceberg_catalog_config import (
+    IcebergCatalogConfig,
+)
+import deltacat.experimental.catalog.iceberg.impl as IcebergCatalog
+
+__all__ = ["IcebergCatalogConfig", "IcebergCatalog"]

deltacat/{catalog → experimental/catalog}/iceberg/iceberg_catalog_config.py

@@ -15,7 +15,7 @@ class IcebergCatalogConfig:
 
     This configuration is passed through to PyIceberg by invoking load_catalog.
     The Properties provided must match properties accepted by PyIceberg for each catalog type
-    See: :func:`deltacat.catalog.iceberg.initialize`
+    See: :func:`deltacat.experimental.catalog.iceberg.initialize`
 
     Attributes:
         type: The PyIceberg Catalog instance

deltacat/{catalog → experimental/catalog}/iceberg/impl.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 
 from typing import Any, Dict, List, Optional, Union
 
@@ -7,13 +8,19 @@ from daft.daft import ScanOperatorHandle, StorageConfig
 from daft.logical.builder import LogicalPlanBuilder
 
 from deltacat import logs
+from deltacat.catalog.model.catalog import Catalog
 from deltacat.catalog.model.table_definition import TableDefinition
-from deltacat.daft.daft_scan import DeltaCatScanOperator
+from deltacat.utils.daft import DeltaCatScanOperator
 from deltacat.exceptions import TableAlreadyExistsError
-from deltacat.storage.iceberg.iceberg_scan_planner import IcebergScanPlanner
-from deltacat.storage.iceberg.model import PartitionSchemeMapper, SchemaMapper
+from deltacat.experimental.storage.iceberg.iceberg_scan_planner import (
+    IcebergScanPlanner,
+)
+from deltacat.experimental.storage.iceberg.model import (
+    PartitionSchemeMapper,
+    SchemaMapper,
+)
 from deltacat.storage.model.partition import PartitionScheme
-from deltacat.storage.iceberg.impl import _get_native_catalog
+from deltacat.experimental.storage.iceberg.impl import _get_native_catalog
 from deltacat.storage.model.sort_key import SortScheme
 from deltacat.storage.model.list_result import ListResult
 from deltacat.storage.model.namespace import Namespace, NamespaceProperties
@@ -26,20 +33,31 @@ from deltacat.storage.model.types import (
     LocalTable,
     StreamFormat,
 )
-from deltacat.storage.iceberg import impl as IcebergStorage
+from deltacat.experimental.storage.iceberg import impl as IcebergStorage
 from deltacat.types.media import ContentType
 from deltacat.types.tables import TableWriteMode
 from deltacat.constants import DEFAULT_NAMESPACE
-from deltacat.catalog.iceberg.iceberg_catalog_config import IcebergCatalogConfig
+from deltacat.experimental.catalog.iceberg.iceberg_catalog_config import (
+    IcebergCatalogConfig,
+)
 
-from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.catalog import Catalog as PyIcebergCatalog, load_catalog
 from pyiceberg.transforms import BucketTransform
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
+IcebergCatalog = sys.modules[__name__]
+
+
+def from_config(config: IcebergCatalogConfig, *args, **kwargs) -> Catalog:
+    """
+    Factory method to construct a catalog from Iceberg catalog configuration.
+    """
+    return Catalog(config, impl=IcebergCatalog, *args, **kwargs)
+
 
 # catalog functions
-def initialize(*args, config: IcebergCatalogConfig, **kwargs) -> Catalog:
+def initialize(config: IcebergCatalogConfig, **kwargs) -> PyIcebergCatalog:
     """
     Initializes an Iceberg catalog with the given config.
 
@@ -123,7 +141,7 @@ def write_to_table(
                     )
                     # TODO(pdames): only append s3:// to output file paths when writing to S3!
                     out_file_paths = [f"s3://{val}" for val in out_df.to_arrow()[0]]
-                    from deltacat.catalog.iceberg import overrides
+                    from deltacat.experimental.catalog.iceberg import overrides
 
                     overrides.append(
                         table_definition.table.native_object,
@@ -180,7 +198,7 @@ def create_table(
     name: str,
     *args,
     namespace: Optional[str] = None,
-    version: Optional[str] = None,
+    table_version: Optional[str] = None,
     lifecycle_state: Optional[LifecycleState] = None,
     schema: Optional[Schema] = None,
     partition_scheme: Optional[PartitionScheme] = None,
@@ -224,7 +242,7 @@ def create_table(
     IcebergStorage.create_table_version(
         namespace=namespace,
         table_name=name,
-        table_version=version,
+        table_version=table_version,
         schema=schema,
         partition_scheme=partition_scheme,
         sort_keys=sort_keys,