deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/experimental/converter_agent/table_monitor.py

@@ -0,0 +1,479 @@
+#!/usr/bin/env python3
+"""
+DeltaCAT Table Monitor Job. Automatically runs data converter sessions in response to table updates.
+"""
+
+import argparse
+import hashlib
+import json
+import logging
+import os
+import time
+from typing import List, Optional
+
+import pyarrow.fs as pafs
+import ray
+import deltacat
+
+from pyiceberg.catalog import load_catalog, CatalogType
+from pyiceberg.exceptions import NoSuchTableError
+
+from deltacat import job_client, local_job_client
+from deltacat.constants import DEFAULT_NAMESPACE
+from deltacat.compute.converter.converter_session import converter_session
+from deltacat.compute.converter.model.converter_session_params import (
+    ConverterSessionParams,
+)
+from deltacat.compute.jobs.client import DeltaCatJobClient
+from deltacat.utils.filesystem import (
+    resolve_path_and_filesystem,
+    FilesystemType,
+)
+import deltacat.logs as logs
+
+# Initialize DeltaCAT logger
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def monitor_table(
+    catalog_type: str,
+    warehouse_path: str,
+    catalog_uri: Optional[str],
+    namespace: str,
+    table_name: str,
+    merge_keys: List[str],
+    filesystem_type: FilesystemType = FilesystemType.LOCAL,
+    monitor_interval: float = 5.0,
+    max_converter_parallelism: int = 1,
+    ray_inactivity_timeout: int = 10,
+) -> None:
+    """Monitor an Iceberg table for changes and run converter sessions when needed."""
+
+    logger.info(
+        f"Starting table monitor. Namespace: '{namespace}', Table: '{table_name}', "
+        f"Warehouse: '{warehouse_path}', Catalog type: '{catalog_type}', "
+        f"Catalog URI: '{catalog_uri or 'None'}', Merge keys: '{merge_keys}', "
+        f"Filesystem type: '{filesystem_type}', Monitor interval: '{monitor_interval}s', "
+        f"Max converter parallelism: '{max_converter_parallelism}', "
+        f"Ray inactivity timeout: '{ray_inactivity_timeout}s'"
+    )
+
+    # Create PyIceberg catalog
+    catalog = load_catalog(
+        "monitor_catalog",
+        type=catalog_type,
+        warehouse=warehouse_path,
+        uri=catalog_uri or None,
+    )
+
+    # Set up filesystem
+    filesystem = FilesystemType.to_filesystem(filesystem_type)
+    if filesystem_type == FilesystemType.UNKNOWN:
+        normalized_warehouse_path, filesystem = resolve_path_and_filesystem(
+            warehouse_path
+        )
+        warehouse_path = normalized_warehouse_path
+
+    logger.info(f"Resolved filesystem: {type(filesystem).__name__}")
+    logger.info(f"Normalized warehouse path: {warehouse_path}")
+
+    # Parse table identifier
+    if not namespace:
+        namespace = DEFAULT_NAMESPACE
+    table_identifier = f"{namespace}.{table_name}"
+    logger.info(f"  - Parsed table - namespace: '{namespace}', table: '{table_name}'")
+
+    last_snapshot_id = None
+    start_time = time.time()
+    last_write_time = start_time  # Track last time we saw table activity
+
+    while True:
+        # Sleep before starting the first iteration and all subsequent iterations
+        logger.debug(f"Sleeping for {monitor_interval}s before next check...")
+        time.sleep(monitor_interval)
+
+        logger.info(f"Checking table {table_identifier} for updates...")
+
+        # Try to load the table
+        try:
+            tbl = catalog.load_table(table_identifier)
+            current_snapshot_id = tbl.metadata.current_snapshot_id
+            if last_snapshot_id != current_snapshot_id:
+                logger.info(
+                    f"New table version detected - snapshot ID: {current_snapshot_id}"
+                )
+                logger.info(f"Table has {len(tbl.metadata.snapshots)} snapshots")
+                logger.info(f"Table format version: {tbl.metadata.format_version}")
+
+                # Update last activity time when we detect table changes
+                last_write_time = time.time()
+
+                # Always run deduplication when there are snapshots (duplicates can exist within a single snapshot)
+                logger.info(
+                    f"Table has data - triggering converter session to resolve any duplicates..."
+                )
+
+                # Run converter session
+                try:
+                    converter_params = ConverterSessionParams.of(
+                        {
+                            "catalog": catalog,
+                            "iceberg_namespace": namespace,
+                            "iceberg_table_name": table_name,
+                            "iceberg_warehouse_bucket_name": warehouse_path,
+                            "merge_keys": merge_keys,
+                            "enforce_primary_key_uniqueness": True,
+                            "task_max_parallelism": max_converter_parallelism,
+                            "filesystem": filesystem,
+                            "location_provider_prefix_override": None,
+                        }
+                    )
+
+                    logger.debug(f"Converter Session Parameters: {converter_params}")
+
+                    logger.info(f"Starting converter session...")
+                    updated_metadata = converter_session(params=converter_params)
+                    logger.info(f"Converter session completed successfully")
+                    current_snapshot_id = updated_metadata.current_snapshot_id
+                    logger.info(
+                        f"Current snapshot ID updated to: {current_snapshot_id}"
+                    )
+                except Exception as e:
+                    logger.error(f"Converter session failed: {e}")
+                    logger.error(f"Exception traceback:", exc_info=True)
+                last_snapshot_id = current_snapshot_id
+            else:
+                logger.debug(
+                    f"No table changes detected (snapshot ID: {current_snapshot_id})"
+                )
+        except NoSuchTableError:
+            logger.info(f"Table {table_identifier} does not exist yet - waiting...")
+        except Exception as e:
+            logger.error(f"Error in table monitor: {e}")
+
+        # Check for Ray inactivity timeout
+        current_time = time.time()
+        inactivity_duration = current_time - last_write_time
+
+        if inactivity_duration >= ray_inactivity_timeout:
+            logger.info(
+                f"Ray inactivity timeout reached ({inactivity_duration:.1f}s >= {ray_inactivity_timeout}s)"
+            )
+            logger.info(
+                f"No table activity detected for {inactivity_duration:.1f} seconds, shutting down Ray..."
+            )
+
+            try:
+                if ray.is_initialized():
+                    ray.shutdown()
+                    logger.info("Ray shutdown successfully due to inactivity")
+                else:
+                    logger.info("Ray was not initialized, nothing to shut down")
+            except Exception as e:
+                logger.error(f"Error shutting down Ray: {e}")
+
+            logger.info(f"Table monitor stopping due to inactivity timeout")
+            break
+
+    logger.info(f"Table monitor completed")
+
+
+def _generate_job_name(warehouse_path: str, namespace: str, table_name: str) -> str:
+    """
+    Generate a unique job name based on warehouse path, namespace, and table name.
+
+    Args:
+        warehouse_path: Warehouse path
+        namespace: Table namespace
+        table_name: Table name
+
+    Returns:
+        Job name string.
+    """
+    # Create a sha1 digest of the warehouse path, namespace, and table name
+    digest = hashlib.sha1(
+        f"{warehouse_path}-{namespace}-{table_name}".encode()
+    ).hexdigest()
+    job_name = f"deltacat-monitor-{digest}"
+
+    return job_name
+
+
+def _cleanup_terminated_jobs_for_submission_id(
+    client: DeltaCatJobClient, submission_id: str
+) -> bool:
+    """Clean up any terminated jobs with the given submission ID."""
+    logger.debug(
+        f"Searching for terminated jobs to cleanup with submission ID: {submission_id}"
+    )
+    try:
+        all_jobs = client.list_jobs()
+        logger.debug(f"All jobs: {all_jobs}")
+        for job in all_jobs:
+            if job.submission_id == submission_id and job.status.is_terminal():
+                logger.info(
+                    f"Cleaning up terminated job: {submission_id} (status: {job.status})"
+                )
+                client.delete_job(submission_id)
+                return True
+    except Exception as e:
+        logger.warning(f"Cleanup failed for job '{submission_id}': {e}")
+    return False
+
+
+def submit_table_monitor_job(
+    warehouse_path: str,
+    catalog_type: CatalogType,
+    catalog_uri: Optional[str],
+    namespace: str,
+    table_name: str,
+    merge_keys: list,
+    monitor_interval: float,
+    max_converter_parallelism: int,
+    filesystem: pafs.FileSystem = None,
+    cluster_cfg_file_path: Optional[str] = None,
+    ray_inactivity_timeout: int = 10,
+) -> str:
+    """
+    Submit a table monitor job to Ray cluster.
+
+    Args:
+        warehouse_path: Warehouse path
+        catalog_type: Catalog type
+        catalog_uri: Catalog URI
+        namespace: Table namespace
+        table_name: Table name to monitor
+        merge_keys: List of merge key column names
+        monitor_interval: Seconds between monitoring checks
+        max_converter_parallelism: Maximum number of concurrent converter tasks
+        filesystem: PyArrow filesystem instance
+        cluster_cfg_file_path: Path to cluster config file (None for local)
+        ray_inactivity_timeout: Seconds to wait before shutting down Ray cluster
+    Returns:
+        Job ID of the submitted job
+    """
+
+    # Parse table identifier to extract namespace and table name
+    if not namespace:
+        namespace = DEFAULT_NAMESPACE
+
+    # Generate unique job ID based on the warehouse and table path
+    job_name = _generate_job_name(
+        warehouse_path=warehouse_path, namespace=namespace, table_name=table_name
+    )
+
+    # Resolve the appropriate local or remote job client
+    if cluster_cfg_file_path:
+        # Submit to remote cluster
+        logger.info(
+            f"Preparing to submit job to remote cluster: {cluster_cfg_file_path}"
+        )
+        # Set the cluster name to the job ID to prevent starting multiple Ray clusters monitoring the same table.
+        client = job_client(cluster_cfg_file_path, cluster_name_override=job_name)
+    else:
+        # Submit to local cluster using DeltaCAT local job client
+        ray_init_args = {
+            "local_mode": True,
+            "resources": {"convert_task": max_converter_parallelism},
+        }
+        logger.info(
+            f"Preparing to submit job locally with ray init args: {ray_init_args}"
+        )
+        client = local_job_client(ray_init_args=ray_init_args)
+
+    # Add filesystem type - determine from filesystem instance
+    filesystem_type = FilesystemType.from_filesystem(filesystem)
+
+    # Build CLI arguments for table_monitor job
+    table_monitor_script_dir = os.path.dirname(os.path.abspath(__file__))
+    table_monitor_script_path = os.path.join(
+        table_monitor_script_dir, "table_monitor.py"
+    )
+
+    logger.debug(f"Table monitor script path: {table_monitor_script_path}")
+    logger.debug(
+        f"Table monitor script exists: {os.path.exists(table_monitor_script_path)}"
+    )
+
+    cmd_args = [
+        f"python {table_monitor_script_path}",
+        f"--catalog-type '{catalog_type.value}'",
+        f"--warehouse-path '{warehouse_path}'",
+        f"--catalog-uri '{catalog_uri}'",
+        f"--namespace '{namespace}'",
+        f"--table-name '{table_name}'",
+        f"--merge-keys '{json.dumps(merge_keys)}'",
+        f"--monitor-interval {monitor_interval}",
+        f"--max-converter-parallelism {max_converter_parallelism}",
+        f"--ray-inactivity-timeout {ray_inactivity_timeout}",
+        f"--filesystem-type '{filesystem_type}'",
+    ]
+
+    # Join all arguments
+    entrypoint = " ".join(cmd_args)
+    logger.debug(
+        f"Submitting table monitor job '{job_name}' with entrypoint: {entrypoint}"
+    )
+
+    # Clean up any terminated jobs with the same submission ID to allow reuse
+    _cleanup_terminated_jobs_for_submission_id(client, job_name)
+
+    # Submit the job with the correct working directory
+    # Working directory should be the converter_agent directory where table_monitor.py is located
+    job_submission_id = client.submit_job(
+        submission_id=job_name,
+        entrypoint=entrypoint,
+        runtime_env={"working_dir": table_monitor_script_dir},
+    )
+
+    logger.info(f"Table monitor job submitted successfully: {job_submission_id}")
+
+    return job_submission_id
+
+
+def run(
+    catalog_type: str,
+    warehouse_path: str,
+    catalog_uri: Optional[str],
+    namespace: str,
+    table_name: str,
+    merge_keys: str,
+    filesystem_type: str = "local",
+    monitor_interval: float = 1.0,
+    max_converter_parallelism: int = 1,
+    ray_inactivity_timeout: int = 10,
+) -> None:
+    """Run table monitor with the given parameters."""
+
+    # Parse merge keys
+    merge_keys_list = json.loads(merge_keys)
+
+    # Run the monitor
+    monitor_table(
+        catalog_type=catalog_type,
+        warehouse_path=warehouse_path,
+        catalog_uri=catalog_uri,
+        namespace=namespace,
+        table_name=table_name,
+        merge_keys=merge_keys_list,
+        filesystem_type=filesystem_type,
+        monitor_interval=monitor_interval,
+        max_converter_parallelism=max_converter_parallelism,
+        ray_inactivity_timeout=ray_inactivity_timeout,
+    )
+
+
+if __name__ == "__main__":
+    """
+    DeltaCAT Table Monitor - Monitor Iceberg tables and run converter sessions
+
+    Example usage:
+    $ python table_monitor.py \
+    $   --catalog-type 'rest' \
+    $   --warehouse-path '/tmp/iceberg-warehouse' \
+    $   --catalog-uri 'http://localhost:8181' \
+    $   --namespace 'default' \
+    $   --table-name 'demo_table' \
+    $   --merge-keys '["id"]' \
+    $   --monitor-interval 1.0 \
+    $   --max-converter-parallelism 2 \
+    $   --ray-inactivity-timeout 300.0
+    """
+
+    script_args = [
+        (
+            ["--catalog-type"],
+            {
+                "help": "Catalog type name (rest, hive, sql)",
+                "type": str,
+                "required": True,
+            },
+        ),
+        (
+            ["--warehouse-path"],
+            {
+                "help": "Warehouse path",
+                "type": str,
+                "required": True,
+            },
+        ),
+        (
+            ["--catalog-uri"],
+            {
+                "help": "Catalog URI",
+                "type": str,
+                "required": True,
+            },
+        ),
+        (
+            ["--namespace"],
+            {
+                "help": "Table namespace",
+                "type": str,
+                "required": True,
+            },
+        ),
+        (
+            ["--table-name"],
+            {
+                "help": "Table name to monitor",
+                "type": str,
+                "required": True,
+            },
+        ),
+        (
+            ["--merge-keys"],
+            {
+                "help": "Comma-separated merge key column names",
+                "type": str,
+                "required": True,
+            },
+        ),
+        (
+            ["--filesystem-type"],
+            {
+                "help": "Filesystem type",
+                "type": str,
+                "default": "local",
+            },
+        ),
+        (
+            ["--monitor-interval"],
+            {
+                "help": "Seconds between monitoring checks",
+                "type": float,
+                "default": 5.0,
+            },
+        ),
+        (
+            ["--max-converter-parallelism"],
+            {
+                "help": "Maximum number of concurrent converter tasks",
+                "type": int,
+                "default": 1,
+            },
+        ),
+        (
+            ["--ray-inactivity-timeout"],
+            {
+                "help": "Ray inactivity timeout in seconds (Ray will shutdown if no activity)",
+                "type": int,
+                "default": 300,
+            },
+        ),
+    ]
+
+    # Parse CLI input arguments
+    parser = argparse.ArgumentParser(
+        description="DeltaCAT Table Monitor - Monitor Iceberg tables and run converter sessions"
+    )
+    for args, kwargs in script_args:
+        parser.add_argument(*args, **kwargs)
+    args = parser.parse_args()
+    print(f"[TABLE MONITOR] Command Line Arguments: {args}")
+
+    # Initialize DeltaCAT
+    deltacat.init()
+
+    # Run the table monitor using the parsed arguments
+    run(**vars(args))

deltacat/experimental/storage/iceberg/iceberg_scan_planner.py

@@ -0,0 +1,129 @@
+import logging
+from typing import Optional, Any, Set
+
+from pyiceberg.catalog import Catalog
+from pyiceberg.table import Table
+import deltacat.logs as logs
+
+from deltacat.storage.model.scan.push_down import Pushdown, PartitionFilter
+from deltacat.storage.model.scan.scan_plan import ScanPlan
+from deltacat.storage.model.scan.scan_task import FileScanTask, DataFile
+from deltacat.storage.util.scan_planner import ScanPlanner
+from deltacat.experimental.storage.iceberg.impl import _try_load_iceberg_table
+from deltacat.experimental.storage.iceberg.visitor import IcebergExpressionVisitor
+
+# Initialize DeltaCAT logger
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+class IcebergScanPlanner(ScanPlanner):
+    def __init__(self, catalog: Catalog):
+        self.catalog = catalog
+        self.expression_visitor = IcebergExpressionVisitor()
+
+    @classmethod
+    def _collect_filter_fields(cls, expr: Any) -> Set[str]:
+        """
+        Collects all field names referenced in the filter expression.
+
+        Args:
+            expr: The expression to analyze
+
+        Returns:
+            Set of field names referenced in the expression
+        """
+        fields = set()
+        if hasattr(expr, "field"):
+            fields.add(expr.field)
+        if hasattr(expr, "left"):
+            fields.update(cls._collect_filter_fields(expr.left))
+        if hasattr(expr, "right"):
+            fields.update(cls._collect_filter_fields(expr.right))
+        if hasattr(expr, "expr"):
+            fields.update(cls._collect_filter_fields(expr.expr))
+        if hasattr(expr, "values"):
+            for value in expr.values:
+                fields.update(cls._collect_filter_fields(value))
+        return fields
+
+    def create_scan_plan(
+        self,
+        table_name: str,
+        namespace: Optional[str] = None,
+        pushdown: Optional[Pushdown] = None,
+    ) -> ScanPlan:
+        iceberg_table = _try_load_iceberg_table(
+            self.catalog, namespace=namespace, table_name=table_name
+        )
+
+        # TODO: implement row, column predicate pushdown to Iceberg
+
+        # Get the partition spec
+        partition_spec = iceberg_table.spec()
+
+        # Check if the table is partitioned
+        is_partitioned = len(partition_spec.fields) > 0
+
+        scan = iceberg_table.scan()
+        if is_partitioned:
+            if pushdown and pushdown.partition_filter:
+                filter_fields = self._collect_filter_fields(pushdown.partition_filter)
+                logger.info(
+                    f"Pushdown partition filter is enabled, converting to Iceberg. Fields discovered in filter: {', '.join(sorted(filter_fields))}"
+                )
+                # Handle partition filter if present, DeltaCAT only supports partition-level filters right now
+                iceberg_expression = self._convert_partition_filter(
+                    iceberg_table, pushdown.partition_filter
+                )
+                scan = scan.filter(iceberg_expression)
+
+        file_scan_tasks = []
+        for scan_task in scan.plan_files():
+            file_scan_tasks.append(FileScanTask([DataFile(scan_task.file.file_path)]))
+        return ScanPlan(file_scan_tasks)
+
+    @classmethod
+    def _validate_partition_references(
+        cls, expr: Any, partition_cols: Set[str]
+    ) -> None:
+        """
+        Validates that the expression only references partition columns.
+
+        Args:
+            expr: The expression to validate
+            partition_cols: Set of valid partition column names
+
+        Raises:
+            ValueError: If the expression references a non-partition column
+        """
+        if hasattr(expr, "field"):  # Reference type expression
+            if expr.field not in partition_cols:
+                raise ValueError(
+                    f"Filter references non-partition column: {expr.field}. "
+                    f"Partition columns are: {partition_cols}"
+                )
+        # Recursively validate nested expressions
+        if hasattr(expr, "left"):
+            cls._validate_partition_references(expr.left, partition_cols)
+        if hasattr(expr, "right"):
+            cls._validate_partition_references(expr.right, partition_cols)
+        if hasattr(expr, "expr"):
+            cls._validate_partition_references(expr.expr, partition_cols)
+        if hasattr(expr, "values"):
+            for value in expr.values:
+                cls._validate_partition_references(value, partition_cols)
+
+    def _convert_partition_filter(
+        self, table: Table, partition_filter: PartitionFilter
+    ):
+        """
+        Convert DeltaCAT partition filter to PyIceberg expression,
+        validating that only partition columns are referenced.
+        """
+        partition_cols = set(field.name for field in table.spec().fields)
+
+        # Validate before converting
+        self._validate_partition_references(partition_filter, partition_cols)
+
+        # Convert to PyIceberg expression
+        return self.expression_visitor.visit(partition_filter)

deltacat/{storage → experimental/storage}/iceberg/impl.py

@@ -32,7 +32,7 @@ from deltacat.storage import (
     NamespaceProperties,
 )
 from deltacat.storage.model.manifest import Manifest
-from deltacat.storage.iceberg.model import (
+from deltacat.experimental.storage.iceberg.model import (
     SchemaMapper,
     PartitionSchemeMapper,
     SortSchemeMapper,
@@ -41,7 +41,7 @@ from deltacat.storage.iceberg.model import (
     NamespaceMapper,
     TableMapper,
 )
-from deltacat.types.media import ContentType, StorageType, TableType
+from deltacat.types.media import ContentType, StorageType, DatasetType
 from deltacat.utils.common import ReadKwargsProvider
 
 from pyiceberg.catalog import Catalog
@@ -281,7 +281,7 @@ def get_latest_delta(
 
 def download_delta(
     delta_like: Union[Delta, DeltaLocator],
-    table_type: TableType = TableType.PYARROW,
+    table_type: DatasetType = DatasetType.PYARROW,
     storage_type: StorageType = StorageType.DISTRIBUTED,
     max_parallelism: Optional[int] = None,
     columns: Optional[List[str]] = None,
@@ -303,7 +303,7 @@ def download_delta(
 def download_delta_manifest_entry(
     delta_like: Union[Delta, DeltaLocator],
     entry_index: int,
-    table_type: TableType = TableType.PYARROW,
+    table_type: DatasetType = DatasetType.PYARROW,
     columns: Optional[List[str]] = None,
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
     *args,
@@ -603,6 +603,8 @@ def stage_delta(
     properties: Optional[DeltaProperties] = None,
     s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
     content_type: ContentType = ContentType.PARQUET,
+    schema: Optional[Schema] = None,
+    sort_scheme_id: Optional[str] = None,
     *args,
     **kwargs,
 ) -> Delta:

deltacat/{storage → experimental/storage}/iceberg/model.py

@@ -66,6 +66,7 @@ from deltacat.storage import (
     TableVersionLocator,
     Transform,
     TransformName,
+    TruncateStrategy,
     TruncateTransform,
     TruncateTransformParameters,
     UnknownTransform,
@@ -227,7 +228,10 @@ class TransformMapper(ModelMapper[IcebergTransform, Transform]):
             )
         if isinstance(obj, IcebergTruncateTransform):
             return TruncateTransform.of(
-                TruncateTransformParameters.of(width=obj.width),
+                TruncateTransformParameters.of(
+                    width=obj.width,
+                    truncate_strategy=TruncateStrategy.ICEBERG,
+                ),
             )
         return UnknownTransform.of()
 
@@ -323,7 +327,7 @@ class PartitionSchemeMapper(ModelMapper[PartitionSpec, PartitionScheme]):
         elif not schema:
             err_msg = "Schema is required for Partition Spec conversion."
             raise ValueError(err_msg)
-        keys = [PartitionKeyMapper.map(field, schema) for field in obj.fields]
+        keys = [PartitionKeyMapper.map(field, schema) for field in obj.fields] or None
         return PartitionScheme.of(
             keys=keys,
             name=name,
@@ -425,7 +429,7 @@ class SortSchemeMapper(ModelMapper[IcebergSortOrder, SortScheme]):
         elif not schema:
             err_msg = "Schema is required for Sort Order conversion."
             raise ValueError(err_msg)
-        keys = [SortKeyMapper.map(field, schema) for field in obj.fields]
+        keys = [SortKeyMapper.map(field, schema) for field in obj.fields] or None
         return SortScheme.of(
             keys=keys,
             name=name,