deltacat 2.0.0b11__py3-none-any.whl → 2.0.0.post1__py3-none-any.whl

deltacat/examples/experimental/iceberg/converter/beam/utils/spark.py

@@ -0,0 +1,263 @@
+"""
+Spark SQL utilities for Iceberg table operations.
+
+This module provides Beam DoFn classes that use Spark SQL to work with Iceberg tables,
+"""
+
+import os
+import apache_beam as beam
+from apache_beam import Row
+
+
+class SparkSQLIcebergRead(beam.DoFn):
+    """
+    Custom Beam DoFn that uses Spark SQL to read Iceberg tables.
+    """
+
+    def __init__(
+        self,
+        table_name: str,
+        catalog_uri: str = "http://localhost:8181",
+        warehouse: str = "warehouse/",
+    ):
+        """
+        Initialize the Spark SQL reader.
+
+        Args:
+            table_name: Name of the Iceberg table
+            catalog_uri: URI of the Iceberg REST catalog
+            warehouse: Warehouse path
+        """
+        self.table_name = table_name
+        self.catalog_uri = catalog_uri
+        self.warehouse = warehouse
+        self.spark = None
+
+    def setup(self):
+        """Set up Spark session (called once per worker)."""
+        try:
+            from pyspark.sql import SparkSession
+            import importlib.metadata
+
+            # Get Spark version for dependency resolution
+            try:
+                spark_version = ".".join(
+                    importlib.metadata.version("pyspark").split(".")[:2]
+                )
+            except Exception:
+                spark_version = "3.5"  # Default fallback
+
+            scala_version = "2.12"
+            iceberg_version = "1.6.0"
+
+            print(f"🔧 Setting up Spark session for reading {self.table_name}")
+            print(f"   - Spark version: {spark_version}")
+            print(f"   - Iceberg version: {iceberg_version}")
+
+            # Set Spark packages for Iceberg runtime
+            os.environ["PYSPARK_SUBMIT_ARGS"] = (
+                f"--packages org.apache.iceberg:iceberg-spark-runtime-{spark_version}_{scala_version}:{iceberg_version} "
+                f"pyspark-shell"
+            )
+
+            # Create Spark session with Iceberg REST catalog configuration
+            self.spark = (
+                SparkSession.builder.appName(f"DeltaCAT Read - {self.table_name}")
+                .config("spark.sql.session.timeZone", "UTC")
+                .config(
+                    "spark.serializer", "org.apache.spark.serializer.KryoSerializer"
+                )
+                .config(
+                    "spark.sql.extensions",
+                    "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+                )
+                # Configure REST catalog
+                .config(
+                    "spark.sql.catalog.rest", "org.apache.iceberg.spark.SparkCatalog"
+                )
+                .config("spark.sql.catalog.rest.type", "rest")
+                .config("spark.sql.catalog.rest.uri", self.catalog_uri)
+                .config("spark.sql.catalog.rest.warehouse", self.warehouse)
+                # Set REST as default catalog
+                .config("spark.sql.defaultCatalog", "rest")
+                # Local mode configuration (within Beam workers)
+                .config("spark.master", "local[1]")  # Single thread per worker
+                .config("spark.sql.adaptive.enabled", "true")
+                # Networking binding
+                .config("spark.driver.bindAddress", "127.0.0.1")
+                .config("spark.driver.host", "127.0.0.1")
+                .config("spark.ui.enabled", "false")
+                .config("spark.sql.adaptive.coalescePartitions.enabled", "false")
+                .getOrCreate()
+            )
+
+            print(f"✅ Spark session created successfully")
+
+        except Exception as e:
+            print(f"❌ Failed to set up Spark session: {e}")
+            raise
+
+    def teardown(self):
+        """Clean up Spark session (called once per worker)."""
+        if self.spark:
+            try:
+                self.spark.stop()
+                print("✅ Spark session stopped")
+            except Exception as e:
+                print(f"⚠️ Error stopping Spark session: {e}")
+
+    def process(self, element):
+        """
+        Process element (read from Iceberg table using Spark SQL).
+
+        Args:
+            element: Input element (not used, just triggers the read)
+
+        Yields:
+            Records from the Iceberg table
+        """
+        try:
+            if not self.spark:
+                raise RuntimeError("Spark session not initialized")
+
+            print(f"📖 Reading table {self.table_name} using Spark SQL")
+
+            # Read from Iceberg table using Spark SQL
+            df = self.spark.sql(f"SELECT * FROM {self.table_name}")
+
+            # Collect all records
+            records = df.collect()
+
+            print(f"📊 Successfully read {len(records)} records from {self.table_name}")
+
+            # Convert Spark rows to Beam Row objects and yield
+            for row in records:
+                row_dict = row.asDict()
+                # Convert to Beam Row for consistency with write mode
+                beam_row = Row(**row_dict)
+                yield beam_row
+
+        except Exception as e:
+            print(f"❌ Failed to read from table {self.table_name}: {e}")
+            raise
+
+
+class SparkSQLIcebergRewrite(beam.DoFn):
+    """
+    Custom Beam DoFn that uses Spark SQL to rewrite Iceberg table data files.
+
+    This uses Spark's rewrite_data_files procedure to materialize positional deletes
+    by rewriting data files. The result is a "clean" table without positional deletes.
+    """
+
+    def __init__(self, catalog_uri, warehouse_path, table_name):
+        self.catalog_uri = catalog_uri
+        self.warehouse_path = warehouse_path
+        self.table_name = table_name
+
+    def setup(self):
+        """Initialize Spark session for rewrite operations."""
+        try:
+            from pyspark.sql import SparkSession
+            import importlib.metadata
+
+            print(f"🔧 Setting up Spark session for rewriting {self.table_name}")
+
+            # Detect Spark version for appropriate Iceberg runtime
+            spark_version = importlib.metadata.version("pyspark")
+            major_minor = ".".join(spark_version.split(".")[:2])
+            print(f"   - Spark version: {major_minor}")
+            print(f"   - Iceberg version: 1.6.0")
+
+            # Configure Spark with Iceberg
+            self.spark = (
+                SparkSession.builder.appName("IcebergRewrite")
+                .config(
+                    "spark.jars.packages",
+                    f"org.apache.iceberg:iceberg-spark-runtime-{major_minor}_2.12:1.6.0",
+                )
+                .config(
+                    "spark.sql.extensions",
+                    "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+                )
+                .config(
+                    "spark.sql.catalog.spark_catalog",
+                    "org.apache.iceberg.spark.SparkSessionCatalog",
+                )
+                .config("spark.sql.catalog.spark_catalog.type", "rest")
+                .config("spark.sql.catalog.spark_catalog.uri", self.catalog_uri)
+                .config(
+                    "spark.sql.catalog.spark_catalog.warehouse", self.warehouse_path
+                )
+                .config("spark.driver.bindAddress", "127.0.0.1")
+                .config("spark.driver.host", "127.0.0.1")
+                .config("spark.ui.enabled", "false")
+                .getOrCreate()
+            )
+
+            print("✅ Spark session created successfully")
+
+        except ImportError as e:
+            raise RuntimeError(
+                f"PySpark is required for rewrite mode. Install with: pip install pyspark"
+            ) from e
+        except Exception as e:
+            raise RuntimeError(f"Failed to create Spark session: {e}") from e
+
+    def process(self, element):
+        """Rewrite table data files to materialize positional deletes."""
+        try:
+            print(
+                f"📋 Rewriting table {self.table_name} to materialize positional deletes"
+            )
+
+            # Use Spark's rewrite_data_files procedure with delete_file_threshold=1
+            # This forces rewrite even when there's only 1 positional delete file
+            rewrite_sql = f"""
+            CALL spark_catalog.system.rewrite_data_files(
+                table => '{self.table_name}',
+                options => map('delete-file-threshold', '1')
+            )
+            """
+
+            print(f"🔄 Executing rewrite procedure with delete_file_threshold=1...")
+            print(f"   SQL: {rewrite_sql.strip()}")
+            print(
+                f"   Rationale: Forces rewrite even with single positional delete file"
+            )
+
+            result = self.spark.sql(rewrite_sql)
+
+            # Collect results to see what was rewritten
+            rewrite_result = result.collect()[0]
+            print(f"📊 Rewrite result: {rewrite_result}")
+
+            # Check if we actually rewrote anything
+            if rewrite_result.rewritten_data_files_count > 0:
+                print(
+                    f"✅ Successfully rewrote {rewrite_result.rewritten_data_files_count} data files"
+                )
+                print(
+                    f"   - Added {rewrite_result.added_data_files_count} new data files"
+                )
+                print(f"   - Rewrote {rewrite_result.rewritten_bytes_count} bytes")
+                print(f"   - Positional deletes have been materialized!")
+            else:
+                print(f"⚠️  No files were rewritten (rewritten_data_files_count=0)")
+                print(f"   - This may indicate no positional deletes exist")
+                print(f"   - Or the table may already be in optimal state")
+
+            yield f"Rewrite completed for {self.table_name}"
+
+        except Exception as e:
+            print(f"❌ Error during rewrite: {e}")
+            import traceback
+
+            traceback.print_exc()
+            yield f"Rewrite failed for {self.table_name}: {e}"
+
+    def teardown(self):
+        """Clean up Spark session."""
+        if hasattr(self, "spark"):
+            print("✅ Spark session stopped")
+            self.spark.stop()

deltacat/exceptions.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 from enum import Enum
-from typing import Callable
+from typing import Callable, Optional, TYPE_CHECKING
 import logging
 
 import tenacity
@@ -28,6 +28,9 @@ from deltacat.utils.ray_utils.runtime import (
     get_current_ray_task_id,
 )
 
+if TYPE_CHECKING:
+    from deltacat.storage.model.schema import FieldLocator
+
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 DELTACAT_STORAGE_PARAM = "deltacat_storage"
@@ -74,9 +77,18 @@ class DeltaCatErrorNames(str, Enum):
     TABLE_NOT_FOUND_ERROR = "TableNotFoundError"
     TABLE_VERSION_NOT_FOUND_ERROR = "TableVersionNotFoundError"
     STREAM_NOT_FOUND_ERROR = "StreamNotFoundError"
+    PARTITION_NOT_FOUND_ERROR = "PartitionNotFoundError"
     DELTA_NOT_FOUND_ERROR = "DeltaNotFoundError"
     TABLE_ALREADY_EXISTS_ERROR = "TableAlreadyExistsError"
+    TABLE_VERSION_ALREADY_EXISTS_ERROR = "TableVersionAlreadyExistsError"
     NAMESPACE_ALREADY_EXISTS_ERROR = "NamespaceAlreadyExistsError"
+    SCHEMA_COMPATIBILITY_ERROR = "SchemaCompatibilityError"
+    SCHEMA_VALIDATION_ERROR = "SchemaValidationError"
+    TABLE_VALIDATION_ERROR = "TableValidationError"
+    CONCURRENT_MODIFICATION_ERROR = "ConcurrentModificationError"
+    OBJECT_NOT_FOUND_ERROR = "ObjectNotFoundError"
+    OBJECT_DELETED_ERROR = "ObjectDeletedError"
+    OBJECT_ALREADY_EXISTS_ERROR = "ObjectAlreadyExistsError"
 
 
 class DeltaCatError(Exception):
@@ -87,9 +99,12 @@ class DeltaCatError(Exception):
         super().__init__(*args, **kwargs)
 
     def _get_ray_task_id_and_node_ip(self):
-        task_id = get_current_ray_task_id()
-        node_ip = ray.util.get_node_ip_address()
-        return task_id, node_ip
+        if ray.is_initialized():
+            task_id = get_current_ray_task_id()
+            node_ip = ray.util.get_node_ip_address()
+            return task_id, node_ip
+        else:
+            return None, None
 
 
 class NonRetryableError(DeltaCatError):
@@ -232,6 +247,10 @@ class TableVersionNotFoundError(NonRetryableError):
     error_name = DeltaCatErrorNames.TABLE_VERSION_NOT_FOUND_ERROR.value
 
 
+class PartitionNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.PARTITION_NOT_FOUND_ERROR.value
+
+
 class StreamNotFoundError(NonRetryableError):
     error_name = DeltaCatErrorNames.STREAM_NOT_FOUND_ERROR.value
 
@@ -244,10 +263,53 @@ class TableAlreadyExistsError(NonRetryableError):
     error_name = DeltaCatErrorNames.TABLE_ALREADY_EXISTS_ERROR.value
 
 
+class TableVersionAlreadyExistsError(NonRetryableError):
+    error_name = DeltaCatErrorNames.TABLE_VERSION_ALREADY_EXISTS_ERROR.value
+
+
 class NamespaceAlreadyExistsError(NonRetryableError):
     error_name = DeltaCatErrorNames.TABLE_ALREADY_EXISTS_ERROR.value
 
 
+class ObjectNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.OBJECT_NOT_FOUND_ERROR.value
+
+
+class ObjectDeletedError(NonRetryableError):
+    error_name = DeltaCatErrorNames.OBJECT_DELETED_ERROR.value
+
+
+class ObjectAlreadyExistsError(NonRetryableError):
+    error_name = DeltaCatErrorNames.OBJECT_ALREADY_EXISTS_ERROR.value
+
+
+class ConcurrentModificationError(NonRetryableError):
+    error_name = DeltaCatErrorNames.CONCURRENT_MODIFICATION_ERROR.value
+
+
+class SchemaValidationError(NonRetryableError):
+    error_name = DeltaCatErrorNames.SCHEMA_VALIDATION_ERROR.value
+
+
+class TableValidationError(NonRetryableError):
+    error_name = DeltaCatErrorNames.TABLE_VALIDATION_ERROR.value
+
+
+class SchemaCompatibilityError(NonRetryableError):
+    error_name = DeltaCatErrorNames.SCHEMA_COMPATIBILITY_ERROR.value
+    """Raised when a schema update would break backward compatibility."""
+
+    def __init__(
+        self,
+        message: str,
+        field_locator: Optional[FieldLocator] = None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(message, *args, **kwargs)
+        self.field_locator = field_locator
+
+
 def categorize_errors(func: Callable):
     def wrapper(*args, **kwargs):
         try:

deltacat/experimental/catalog/iceberg/impl.py

@@ -198,7 +198,7 @@ def create_table(
     name: str,
     *args,
     namespace: Optional[str] = None,
-    version: Optional[str] = None,
+    table_version: Optional[str] = None,
     lifecycle_state: Optional[LifecycleState] = None,
     schema: Optional[Schema] = None,
     partition_scheme: Optional[PartitionScheme] = None,
@@ -242,7 +242,7 @@ def create_table(
     IcebergStorage.create_table_version(
         namespace=namespace,
         table_name=name,
-        table_version=version,
+        table_version=table_version,
         schema=schema,
         partition_scheme=partition_scheme,
         sort_keys=sort_keys,

deltacat/experimental/compatibility/backfill_locator_to_id_mappings.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""
+Backfill script for backwards compatibility with canonical_string changes.
+
+This script migrates existing DeltaCAT catalogs from the old global canonical string
+format (with parent hexdigest) to the new hierarchical format (without parent hexdigest).
+
+The old format was: {parent_hexdigest}|{name_parts}
+The new format is: {name_parts}
+
+Strategy:
+1. Patch canonical_string method to use old format for reading existing name mappings
+2. Use dc.list() to recursively discover all objects with old canonical_string
+3. Copy each object's name mappings using new canonical_string format for writing
+4. Works with any PyArrow-supported filesystem (local, S3, GCS, etc.)
+
+Usage:
+    python deltacat/experimental/compatibility/backfill_locator_to_id_mappings.py --catalog-root /path/to/catalog
+"""
+
+import argparse
+import logging
+import contextlib
+
+import deltacat as dc
+from deltacat.utils.url import DeltaCatUrl
+from deltacat.storage.model.locator import Locator
+from deltacat.api import _copy_objects_in_order
+
+
+logger = logging.getLogger(__name__)
+
+
+def canonical_string_old(locator, separator: str = "|") -> str:
+    """
+    Old implementation of canonical_string that included parent hexdigest.
+    This is used to read existing name resolution directories.
+    """
+    parts = []
+    parent_hexdigest = locator.parent.hexdigest() if locator.parent else None
+    if parent_hexdigest:
+        parts.append(parent_hexdigest)
+    parts.extend(locator.name.parts())
+    return separator.join([str(part) for part in parts])
+
+
+@contextlib.contextmanager
+def patched_canonical_string(use_old_format: bool = True):
+    """
+    Context manager that temporarily patches the canonical_string method.
+
+    Args:
+        use_old_format: If True, use old format; if False, use new format
+    """
+    # Store original method
+    original_method = Locator.canonical_string
+
+    try:
+        if use_old_format:
+            # Patch with old implementation
+            Locator.canonical_string = canonical_string_old
+        # If use_old_format is False, keep the current (new) implementation
+
+        yield
+
+    finally:
+        # Always restore original method
+        Locator.canonical_string = original_method
+
+
+def migrate_catalog(
+    source_url: str, destination_url: str, dry_run: bool = False
+) -> bool:
+    """
+    Migrate a catalog from old to new canonical string format.
+
+    Args:
+        source_url: Source catalog URL (e.g., 'dc://catalog_root/')
+        destination_url: Destination catalog URL (e.g., 'dc://new_catalog_root/')
+        dry_run: If True, just show what would be migrated
+
+    Returns:
+        True if migration successful, False otherwise
+    """
+    try:
+        src_url = DeltaCatUrl(source_url)
+        dst_url = DeltaCatUrl(destination_url)
+
+        logger.info(f"Starting migration from {source_url} to {destination_url}")
+
+        if dry_run:
+            logger.info("DRY RUN - No actual changes will be made")
+
+        if dry_run:
+            # Step 1: List all objects using old canonical_string format for dry run
+            logger.info(
+                "DRY RUN - Discovering objects using old canonical string format..."
+            )
+            with patched_canonical_string(use_old_format=True):
+                src_objects = dc.list(src_url, recursive=True)
+
+            if hasattr(src_objects, "__len__"):
+                logger.info(f"DRY RUN - Found {len(src_objects)} objects to migrate")
+            else:
+                logger.info("DRY RUN - Found objects to migrate (count unknown)")
+
+            logger.info(
+                "DRY RUN - Would copy objects using new canonical string format"
+            )
+            return True
+
+        # Step 2: Read objects with old format, then write with new format
+        logger.info("Step 1: Reading all objects using old canonical string format...")
+        with patched_canonical_string(use_old_format=True):
+            src_objects = dc.list(src_url, recursive=True)
+
+        if hasattr(src_objects, "__len__"):
+            logger.info(f"Found {len(src_objects)} objects to migrate")
+        else:
+            logger.info("Found objects to migrate (count unknown)")
+
+        logger.info("Step 2: Writing objects using new canonical string format...")
+        with patched_canonical_string(use_old_format=False):
+            _copy_objects_in_order(src_objects, dst_url)
+
+        logger.info("Migration completed successfully!")
+        return True
+
+    except Exception as e:
+        logger.error(f"Migration failed: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Backfill locator-to-ID mappings for DeltaCAT canonical string changes"
+    )
+    parser.add_argument(
+        "--catalog-root",
+        required=True,
+        help="Path to the source DeltaCAT catalog root directory",
+    )
+    parser.add_argument(
+        "--destination",
+        required=True,
+        help="Path to the destination DeltaCAT catalog root directory",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show what would be migrated without making changes",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Enable verbose logging. Writes logs to /tmp/deltacat/ by default.",
+    )
+
+    args = parser.parse_args()
+
+    # Configure logging
+    level = logging.DEBUG if args.verbose else logging.INFO
+    logging.basicConfig(level=level, format="%(asctime)s - %(levelname)s - %(message)s")
+
+    # Initialize DeltaCAT with the catalog
+    catalog_config = {
+        "local": {
+            "root": args.catalog_root,
+        }
+    }
+    dc.init(catalogs=catalog_config)
+
+    try:
+        # Migrate to different location
+        source_url = f"dc://{args.catalog_root}/"
+        dest_url = f"dc://{args.destination}/"
+
+        if not args.dry_run:
+            # Initialize destination catalog
+            dest_config = {
+                "dest": {
+                    "root": args.destination,
+                }
+            }
+            dc.init(catalogs=dest_config)
+
+        success = migrate_catalog(source_url, dest_url, args.dry_run)
+
+        return int(success)
+
+    except Exception as e:
+        logger.error(f"Migration failed: {e}")
+        return 1
+
+
+if __name__ == "__main__":
+    exit(main())