deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/experimental/compatibility/backfill_locator_to_id_mappings.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""
+Backfill script for backwards compatibility with canonical_string changes.
+
+This script migrates existing DeltaCAT catalogs from the old global canonical string
+format (with parent hexdigest) to the new hierarchical format (without parent hexdigest).
+
+The old format was: {parent_hexdigest}|{name_parts}
+The new format is: {name_parts}
+
+Strategy:
+1. Patch canonical_string method to use old format for reading existing name mappings
+2. Use dc.list() to recursively discover all objects with old canonical_string
+3. Copy each object's name mappings using new canonical_string format for writing
+4. Works with any PyArrow-supported filesystem (local, S3, GCS, etc.)
+
+Usage:
+    python deltacat/experimental/compatibility/backfill_locator_to_id_mappings.py --catalog-root /path/to/catalog
+"""
+
+import argparse
+import logging
+import contextlib
+
+import deltacat as dc
+from deltacat.utils.url import DeltaCatUrl
+from deltacat.storage.model.locator import Locator
+from deltacat.api import _copy_objects_in_order
+
+
+logger = logging.getLogger(__name__)
+
+
+def canonical_string_old(locator, separator: str = "|") -> str:
+    """
+    Old implementation of canonical_string that included parent hexdigest.
+    This is used to read existing name resolution directories.
+    """
+    parts = []
+    parent_hexdigest = locator.parent.hexdigest() if locator.parent else None
+    if parent_hexdigest:
+        parts.append(parent_hexdigest)
+    parts.extend(locator.name.parts())
+    return separator.join([str(part) for part in parts])
+
+
+@contextlib.contextmanager
+def patched_canonical_string(use_old_format: bool = True):
+    """
+    Context manager that temporarily patches the canonical_string method.
+
+    Args:
+        use_old_format: If True, use old format; if False, use new format
+    """
+    # Store original method
+    original_method = Locator.canonical_string
+
+    try:
+        if use_old_format:
+            # Patch with old implementation
+            Locator.canonical_string = canonical_string_old
+        # If use_old_format is False, keep the current (new) implementation
+
+        yield
+
+    finally:
+        # Always restore original method
+        Locator.canonical_string = original_method
+
+
+def migrate_catalog(
+    source_url: str, destination_url: str, dry_run: bool = False
+) -> bool:
+    """
+    Migrate a catalog from old to new canonical string format.
+
+    Args:
+        source_url: Source catalog URL (e.g., 'dc://catalog_root/')
+        destination_url: Destination catalog URL (e.g., 'dc://new_catalog_root/')
+        dry_run: If True, just show what would be migrated
+
+    Returns:
+        True if migration successful, False otherwise
+    """
+    try:
+        src_url = DeltaCatUrl(source_url)
+        dst_url = DeltaCatUrl(destination_url)
+
+        logger.info(f"Starting migration from {source_url} to {destination_url}")
+
+        if dry_run:
+            logger.info("DRY RUN - No actual changes will be made")
+
+        if dry_run:
+            # Step 1: List all objects using old canonical_string format for dry run
+            logger.info(
+                "DRY RUN - Discovering objects using old canonical string format..."
+            )
+            with patched_canonical_string(use_old_format=True):
+                src_objects = dc.list(src_url, recursive=True)
+
+            if hasattr(src_objects, "__len__"):
+                logger.info(f"DRY RUN - Found {len(src_objects)} objects to migrate")
+            else:
+                logger.info("DRY RUN - Found objects to migrate (count unknown)")
+
+            logger.info(
+                "DRY RUN - Would copy objects using new canonical string format"
+            )
+            return True
+
+        # Step 2: Read objects with old format, then write with new format
+        logger.info("Step 1: Reading all objects using old canonical string format...")
+        with patched_canonical_string(use_old_format=True):
+            src_objects = dc.list(src_url, recursive=True)
+
+        if hasattr(src_objects, "__len__"):
+            logger.info(f"Found {len(src_objects)} objects to migrate")
+        else:
+            logger.info("Found objects to migrate (count unknown)")
+
+        logger.info("Step 2: Writing objects using new canonical string format...")
+        with patched_canonical_string(use_old_format=False):
+            _copy_objects_in_order(src_objects, dst_url)
+
+        logger.info("Migration completed successfully!")
+        return True
+
+    except Exception as e:
+        logger.error(f"Migration failed: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Backfill locator-to-ID mappings for DeltaCAT canonical string changes"
+    )
+    parser.add_argument(
+        "--catalog-root",
+        required=True,
+        help="Path to the source DeltaCAT catalog root directory",
+    )
+    parser.add_argument(
+        "--destination",
+        required=True,
+        help="Path to the destination DeltaCAT catalog root directory",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show what would be migrated without making changes",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Enable verbose logging. Writes logs to /tmp/deltacat/ by default.",
+    )
+
+    args = parser.parse_args()
+
+    # Configure logging
+    level = logging.DEBUG if args.verbose else logging.INFO
+    logging.basicConfig(level=level, format="%(asctime)s - %(levelname)s - %(message)s")
+
+    # Initialize DeltaCAT with the catalog
+    catalog_config = {
+        "local": {
+            "root": args.catalog_root,
+        }
+    }
+    dc.init(catalogs=catalog_config)
+
+    try:
+        # Migrate to different location
+        source_url = f"dc://{args.catalog_root}/"
+        dest_url = f"dc://{args.destination}/"
+
+        if not args.dry_run:
+            # Initialize destination catalog
+            dest_config = {
+                "dest": {
+                    "root": args.destination,
+                }
+            }
+            dc.init(catalogs=dest_config)
+
+        success = migrate_catalog(source_url, dest_url, args.dry_run)
+
+        return int(success)
+
+    except Exception as e:
+        logger.error(f"Migration failed: {e}")
+        return 1
+
+
+if __name__ == "__main__":
+    exit(main())

deltacat/experimental/converter_agent/beam/managed.py

@@ -0,0 +1,173 @@
+"""
+DeltaCAT Job-based Managed I/O for Apache Beam
+
+This module provides a job-based implementation of the DeltaCAT table monitor
+that uses Ray jobs for better scalability and resource management instead of
+threading.
+
+Key Features:
+- Uses DeltaCAT jobs for table monitoring
+- Unique job IDs prevent duplicate monitoring jobs
+- Supports both local and remote Ray clusters
+- Backward compatible with existing managed.py interface
+"""
+
+import logging
+from typing import Dict, Any
+
+import apache_beam as beam
+from pyiceberg.catalog import CatalogType
+
+from deltacat.experimental.converter_agent.table_monitor import submit_table_monitor_job
+from deltacat.compute.converter.constants import DEFAULT_CONVERTER_TASK_MAX_PARALLELISM
+import deltacat.logs as logs
+
+# Initialize DeltaCAT logger
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+# Store original functions before monkey-patching
+_original_write = beam.managed.Write
+
+
+# Create a dictionary of Java catalog impl to CatalogType
+JAVA_ICEBERG_CATALOG_IMPL_TO_TYPE = {
+    "org.apache.iceberg.rest.restcatalog": CatalogType.REST,
+    "org.apache.iceberg.hive.hivecatalog": CatalogType.HIVE,
+    "org.apache.iceberg.aws.glue.gluecatalog": CatalogType.GLUE,
+    "org.apache.iceberg.jdbc.jdbccatalog": CatalogType.SQL,
+}
+
+
+def _extract_catalog_config_from_beam(config: Dict[str, Any]) -> Dict[str, Any]:
+    """Extract catalog configuration from Beam config."""
+    catalog_properties = config.get("catalog_properties", {})
+
+    # Extract catalog implementation class
+    catalog_impl = catalog_properties.get("catalog-impl")
+
+    # Extract catalog type
+    catalog_type = catalog_properties.get("type")
+
+    # Extract other relevant properties
+    warehouse = catalog_properties.get("warehouse", "")
+    uri = catalog_properties.get("uri", "")
+
+    return {
+        "catalog_impl": catalog_impl,
+        "type": catalog_type,
+        "warehouse": warehouse,
+        "uri": uri,
+        "catalog_properties": catalog_properties,
+    }
+
+
+def write(*args, **kwargs):
+    """Wrapper over beam.managed.Write that automatically creates a DeltaCAT table monitor & converter job."""
+    logger.debug(f"Starting DeltaCAT write operation")
+    logger.debug(f"args: {args}")
+    logger.debug(f"kwargs keys: {list(kwargs.keys()) if kwargs else 'None'}")
+
+    # Extract and pop deltacat-specific config keys
+    config = kwargs.get("config", {}).copy() if kwargs.get("config") else {}
+
+    # Extract DeltaCAT converter properties from parent config or individual keys (for backward compatibility)
+    deltacat_converter_properties = config.pop("deltacat_converter_properties", {})
+
+    # Support both new nested structure and old flat structure for backward compatibility
+    deltacat_converter_interval = deltacat_converter_properties.get(
+        "deltacat_converter_interval", 3.0
+    )
+
+    merge_keys = deltacat_converter_properties.get("merge_keys")
+
+    # Extract filesystem parameter (optional) - can be in converter properties or top-level config
+    filesystem = deltacat_converter_properties.get("filesystem", None)
+
+    # Extract cluster configuration file path (for remote jobs)
+    cluster_cfg_file_path = deltacat_converter_properties.get(
+        "cluster_cfg_file_path", None
+    )
+
+    # Extract max converter parallelism
+    max_converter_parallelism = deltacat_converter_properties.get(
+        "max_converter_parallelism",
+        DEFAULT_CONVERTER_TASK_MAX_PARALLELISM,
+    )
+
+    # Extract ray inactivity timeout
+    ray_inactivity_timeout = deltacat_converter_properties.get(
+        "ray_inactivity_timeout", 10
+    )
+
+    # Extract table identifier and warehouse path
+    table_identifier = config.get("table")
+    if not table_identifier:
+        raise ValueError("Table is required")
+
+    if table_identifier and "." in table_identifier:
+        namespace, table_name = table_identifier.split(".", 1)
+    else:
+        namespace = "default"
+        table_name = table_identifier
+
+    warehouse_path = config.get("catalog_properties", {}).get("warehouse", "")
+
+    # Extract catalog configuration for monitoring
+    beam_catalog_config = _extract_catalog_config_from_beam(config)
+
+    # Derive CatalogType from "catalog_impl" or "type" property
+    catalog_impl = beam_catalog_config.get("catalog_impl")
+    if catalog_impl:
+        catalog_type = JAVA_ICEBERG_CATALOG_IMPL_TO_TYPE.get(catalog_impl.lower())
+        if not catalog_type:
+            raise ValueError(f"Unsupported catalog implementation: {catalog_impl}")
+    else:
+        catalog_type_str = beam_catalog_config.get("type")
+        if catalog_type_str:
+            catalog_type = CatalogType(catalog_type_str.lower())
+        else:
+            raise ValueError(
+                f"No catalog implementation or type found in config: {beam_catalog_config}"
+            )
+
+    # Update kwargs with the modified config
+    if "config" in kwargs:
+        kwargs["config"] = config
+
+    logger.debug(f"Preparing to submit table monitor job...")
+    logger.debug(f"table_name: {table_name}")
+    logger.debug(f"deltacat_converter_interval: {deltacat_converter_interval}s")
+    logger.debug(f"merge_keys: {merge_keys}")
+    logger.debug(f"warehouse_path: {warehouse_path}")
+    logger.debug(
+        f"filesystem: {type(filesystem).__name__ if filesystem else 'None (auto-resolve)'}"
+    )
+    logger.debug(f"cluster_cfg_file_path: {cluster_cfg_file_path or 'None (local)'}")
+    logger.debug(f"max_converter_parallelism: {max_converter_parallelism}")
+    logger.debug(f"ray_inactivity_timeout: {ray_inactivity_timeout}s")
+    logger.debug(
+        f"using deltacat_converter_properties: {len(deltacat_converter_properties) > 0}"
+    )
+    logger.debug(f"catalog_type: {catalog_type}")
+
+    # Submit monitoring job
+    try:
+        submit_table_monitor_job(
+            warehouse_path=warehouse_path,
+            catalog_type=catalog_type,
+            catalog_uri=beam_catalog_config.get("uri"),
+            namespace=namespace,
+            table_name=table_name,
+            merge_keys=merge_keys,
+            monitor_interval=deltacat_converter_interval,
+            filesystem=filesystem,
+            cluster_cfg_file_path=cluster_cfg_file_path,
+            max_converter_parallelism=max_converter_parallelism,
+            ray_inactivity_timeout=ray_inactivity_timeout,
+        )
+    except Exception as e:
+        # Don't fail the write operation, just log the error
+        logger.error(f"Failed to submit table monitor job: {e}")
+        logger.error(f"Exception traceback:", exc_info=True)
+    logger.info(f"Delegating to beam.managed.Write")
+    return _original_write(*args, **kwargs)