deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/examples/experimental/iceberg/converter/beam/utils/spark.py

@@ -0,0 +1,263 @@
+"""
+Spark SQL utilities for Iceberg table operations.
+
+This module provides Beam DoFn classes that use Spark SQL to work with Iceberg tables,
+"""
+
+import os
+import apache_beam as beam
+from apache_beam import Row
+
+
+class SparkSQLIcebergRead(beam.DoFn):
+    """
+    Custom Beam DoFn that uses Spark SQL to read Iceberg tables.
+    """
+
+    def __init__(
+        self,
+        table_name: str,
+        catalog_uri: str = "http://localhost:8181",
+        warehouse: str = "warehouse/",
+    ):
+        """
+        Initialize the Spark SQL reader.
+
+        Args:
+            table_name: Name of the Iceberg table
+            catalog_uri: URI of the Iceberg REST catalog
+            warehouse: Warehouse path
+        """
+        self.table_name = table_name
+        self.catalog_uri = catalog_uri
+        self.warehouse = warehouse
+        self.spark = None
+
+    def setup(self):
+        """Set up Spark session (called once per worker)."""
+        try:
+            from pyspark.sql import SparkSession
+            import importlib.metadata
+
+            # Get Spark version for dependency resolution
+            try:
+                spark_version = ".".join(
+                    importlib.metadata.version("pyspark").split(".")[:2]
+                )
+            except Exception:
+                spark_version = "3.5"  # Default fallback
+
+            scala_version = "2.12"
+            iceberg_version = "1.6.0"
+
+            print(f"🔧 Setting up Spark session for reading {self.table_name}")
+            print(f"   - Spark version: {spark_version}")
+            print(f"   - Iceberg version: {iceberg_version}")
+
+            # Set Spark packages for Iceberg runtime
+            os.environ["PYSPARK_SUBMIT_ARGS"] = (
+                f"--packages org.apache.iceberg:iceberg-spark-runtime-{spark_version}_{scala_version}:{iceberg_version} "
+                f"pyspark-shell"
+            )
+
+            # Create Spark session with Iceberg REST catalog configuration
+            self.spark = (
+                SparkSession.builder.appName(f"DeltaCAT Read - {self.table_name}")
+                .config("spark.sql.session.timeZone", "UTC")
+                .config(
+                    "spark.serializer", "org.apache.spark.serializer.KryoSerializer"
+                )
+                .config(
+                    "spark.sql.extensions",
+                    "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+                )
+                # Configure REST catalog
+                .config(
+                    "spark.sql.catalog.rest", "org.apache.iceberg.spark.SparkCatalog"
+                )
+                .config("spark.sql.catalog.rest.type", "rest")
+                .config("spark.sql.catalog.rest.uri", self.catalog_uri)
+                .config("spark.sql.catalog.rest.warehouse", self.warehouse)
+                # Set REST as default catalog
+                .config("spark.sql.defaultCatalog", "rest")
+                # Local mode configuration (within Beam workers)
+                .config("spark.master", "local[1]")  # Single thread per worker
+                .config("spark.sql.adaptive.enabled", "true")
+                # Networking binding
+                .config("spark.driver.bindAddress", "127.0.0.1")
+                .config("spark.driver.host", "127.0.0.1")
+                .config("spark.ui.enabled", "false")
+                .config("spark.sql.adaptive.coalescePartitions.enabled", "false")
+                .getOrCreate()
+            )
+
+            print(f"✅ Spark session created successfully")
+
+        except Exception as e:
+            print(f"❌ Failed to set up Spark session: {e}")
+            raise
+
+    def teardown(self):
+        """Clean up Spark session (called once per worker)."""
+        if self.spark:
+            try:
+                self.spark.stop()
+                print("✅ Spark session stopped")
+            except Exception as e:
+                print(f"⚠️ Error stopping Spark session: {e}")
+
+    def process(self, element):
+        """
+        Process element (read from Iceberg table using Spark SQL).
+
+        Args:
+            element: Input element (not used, just triggers the read)
+
+        Yields:
+            Records from the Iceberg table
+        """
+        try:
+            if not self.spark:
+                raise RuntimeError("Spark session not initialized")
+
+            print(f"📖 Reading table {self.table_name} using Spark SQL")
+
+            # Read from Iceberg table using Spark SQL
+            df = self.spark.sql(f"SELECT * FROM {self.table_name}")
+
+            # Collect all records
+            records = df.collect()
+
+            print(f"📊 Successfully read {len(records)} records from {self.table_name}")
+
+            # Convert Spark rows to Beam Row objects and yield
+            for row in records:
+                row_dict = row.asDict()
+                # Convert to Beam Row for consistency with write mode
+                beam_row = Row(**row_dict)
+                yield beam_row
+
+        except Exception as e:
+            print(f"❌ Failed to read from table {self.table_name}: {e}")
+            raise
+
+
+class SparkSQLIcebergRewrite(beam.DoFn):
+    """
+    Custom Beam DoFn that uses Spark SQL to rewrite Iceberg table data files.
+
+    This uses Spark's rewrite_data_files procedure to materialize positional deletes
+    by rewriting data files. The result is a "clean" table without positional deletes.
+    """
+
+    def __init__(self, catalog_uri, warehouse_path, table_name):
+        self.catalog_uri = catalog_uri
+        self.warehouse_path = warehouse_path
+        self.table_name = table_name
+
+    def setup(self):
+        """Initialize Spark session for rewrite operations."""
+        try:
+            from pyspark.sql import SparkSession
+            import importlib.metadata
+
+            print(f"🔧 Setting up Spark session for rewriting {self.table_name}")
+
+            # Detect Spark version for appropriate Iceberg runtime
+            spark_version = importlib.metadata.version("pyspark")
+            major_minor = ".".join(spark_version.split(".")[:2])
+            print(f"   - Spark version: {major_minor}")
+            print(f"   - Iceberg version: 1.6.0")
+
+            # Configure Spark with Iceberg
+            self.spark = (
+                SparkSession.builder.appName("IcebergRewrite")
+                .config(
+                    "spark.jars.packages",
+                    f"org.apache.iceberg:iceberg-spark-runtime-{major_minor}_2.12:1.6.0",
+                )
+                .config(
+                    "spark.sql.extensions",
+                    "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+                )
+                .config(
+                    "spark.sql.catalog.spark_catalog",
+                    "org.apache.iceberg.spark.SparkSessionCatalog",
+                )
+                .config("spark.sql.catalog.spark_catalog.type", "rest")
+                .config("spark.sql.catalog.spark_catalog.uri", self.catalog_uri)
+                .config(
+                    "spark.sql.catalog.spark_catalog.warehouse", self.warehouse_path
+                )
+                .config("spark.driver.bindAddress", "127.0.0.1")
+                .config("spark.driver.host", "127.0.0.1")
+                .config("spark.ui.enabled", "false")
+                .getOrCreate()
+            )
+
+            print("✅ Spark session created successfully")
+
+        except ImportError as e:
+            raise RuntimeError(
+                f"PySpark is required for rewrite mode. Install with: pip install pyspark"
+            ) from e
+        except Exception as e:
+            raise RuntimeError(f"Failed to create Spark session: {e}") from e
+
+    def process(self, element):
+        """Rewrite table data files to materialize positional deletes."""
+        try:
+            print(
+                f"📋 Rewriting table {self.table_name} to materialize positional deletes"
+            )
+
+            # Use Spark's rewrite_data_files procedure with delete_file_threshold=1
+            # This forces rewrite even when there's only 1 positional delete file
+            rewrite_sql = f"""
+            CALL spark_catalog.system.rewrite_data_files(
+                table => '{self.table_name}',
+                options => map('delete-file-threshold', '1')
+            )
+            """
+
+            print(f"🔄 Executing rewrite procedure with delete_file_threshold=1...")
+            print(f"   SQL: {rewrite_sql.strip()}")
+            print(
+                f"   Rationale: Forces rewrite even with single positional delete file"
+            )
+
+            result = self.spark.sql(rewrite_sql)
+
+            # Collect results to see what was rewritten
+            rewrite_result = result.collect()[0]
+            print(f"📊 Rewrite result: {rewrite_result}")
+
+            # Check if we actually rewrote anything
+            if rewrite_result.rewritten_data_files_count > 0:
+                print(
+                    f"✅ Successfully rewrote {rewrite_result.rewritten_data_files_count} data files"
+                )
+                print(
+                    f"   - Added {rewrite_result.added_data_files_count} new data files"
+                )
+                print(f"   - Rewrote {rewrite_result.rewritten_bytes_count} bytes")
+                print(f"   - Positional deletes have been materialized!")
+            else:
+                print(f"⚠️  No files were rewritten (rewritten_data_files_count=0)")
+                print(f"   - This may indicate no positional deletes exist")
+                print(f"   - Or the table may already be in optimal state")
+
+            yield f"Rewrite completed for {self.table_name}"
+
+        except Exception as e:
+            print(f"❌ Error during rewrite: {e}")
+            import traceback
+
+            traceback.print_exc()
+            yield f"Rewrite failed for {self.table_name}: {e}"
+
+    def teardown(self):
+        """Clean up Spark session."""
+        if hasattr(self, "spark"):
+            print("✅ Spark session stopped")
+            self.spark.stop()

deltacat/examples/experimental/iceberg/iceberg_bucket_writer.py

@@ -0,0 +1,184 @@
+import os
+import logging
+
+import uuid
+import daft
+from pyiceberg.catalog import CatalogType
+
+import deltacat as dc
+
+from deltacat import logs
+from deltacat import IcebergCatalog
+from deltacat.experimental.catalog.iceberg import IcebergCatalogConfig
+from env import store_cli_args_in_os_environ
+
+from pyiceberg.schema import (
+    Schema,
+    NestedField,
+    DoubleType,
+    StringType,
+)
+from pyiceberg.partitioning import PartitionSpec, PartitionField
+from pyiceberg.transforms import BucketTransform
+
+from deltacat.experimental.storage.iceberg.model import (
+    SchemaMapper,
+    PartitionSchemeMapper,
+)
+from deltacat.env import create_ray_runtime_environment
+
+# initialize the driver logger
+driver_logger = logs.configure_application_logger(logging.getLogger(__name__))
+
+
+def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
+    """
+    This is an e2e example that
+        1. creates a DeltaCAT Table (backed by an Iceberg Table) in Glue
+        2. writes data into the DeltaCAT Table
+        3. reads data from the DeltaCAT Table using Daft
+
+    To run the script:
+        1. prepare an AWS Account
+            1. prepare a S3 location where the data will be written to, which will be used in Step 3.
+            2. prepare an IAM Role that has access to the S3 location and Glue
+        2. retrieve the IAM Role AWS Credential and cache locally in ~/.aws/credentials
+        3. run below command to execute the example
+        ```
+        make venv && source venv/bin/activate
+        python -m deltacat.examples.iceberg.iceberg_bucket_writer --warehouse=s3://<YOUR_S3_LOCATION>
+        ```
+
+    """
+    # create any runtime environment required to run the example
+    runtime_env = create_ray_runtime_environment()
+
+    # Start by initializing DeltaCAT and registering available Catalogs.
+    # Ray will be initialized automatically via `ray.init()`.
+    # Only the `iceberg` data catalog is provided so it will become the default.
+    # If initializing multiple catalogs, use the `default_catalog_name` param
+    # to specify which catalog should be the default.
+
+    dc.init(
+        catalogs={
+            # the name of the DeltaCAT catalog is "iceberg"
+            "iceberg": dc.Catalog(
+                # Apache Iceberg implementation of deltacat.catalog.interface
+                impl=IcebergCatalog,
+                # kwargs for pyiceberg.catalog.load_catalog start here...
+                # the name of the Iceberg catalog is "example-iceberg-catalog"
+                name="example-iceberg-catalog",
+                # for additional properties see:
+                # https://py.iceberg.apache.org/configuration/
+                config=IcebergCatalogConfig(
+                    type=CatalogType.GLUE,
+                    properties={
+                        "warehouse": warehouse,
+                        "region_name": "us-east-1",
+                    },
+                ),
+            )
+        },
+        # pass the runtime environment into ray.init()
+        ray_init_args={"runtime_env": runtime_env},
+    )
+
+    # define a native Iceberg table schema
+    schema = Schema(
+        NestedField(field_id=1, name="symbol", field_type=StringType(), required=True),
+        NestedField(field_id=2, name="bid", field_type=DoubleType(), required=False),
+        NestedField(field_id=3, name="ask", field_type=DoubleType(), required=False),
+    )
+
+    # define a native Iceberg partition spec
+    partition_spec = PartitionSpec(
+        PartitionField(
+            source_id=1,
+            field_id=1000,
+            transform=BucketTransform(2),
+            name="symbol_bucket",
+        )
+    )
+
+    # define a native Iceberg sort order
+    # sort_order = SortOrder(SortField(source_id=1, transform=IdentityTransform()))
+
+    # define the Daft dataframe to write
+    df = daft.from_pydict(
+        {
+            "symbol": ["amzn", "goog", "meta", "msft"],
+            "bid": [157.16, 150.55, 392.03, 403.25],
+            "ask": [157.17, 150.56, 392.09, 403.27],
+        }
+    )
+
+    # write to a table named `test_namespace.test_table_bucketed-<SUFFIX>`
+    # we don't need to specify which catalog to create this table in since
+    # only the "iceberg" catalog is available
+    table_name = f"test_table_bucketed-{uuid.uuid4().hex[:8]}"
+    namespace = "test_namespace"
+    print(f"Creating Glue Table: {namespace}.{table_name}")
+    dc.write_to_table(
+        data=df,
+        # path=warehouse + "/datafiles",
+        table=table_name,
+        namespace=namespace,
+        schema=SchemaMapper.map(schema),
+        partition_scheme=PartitionSchemeMapper.map(partition_spec, schema),
+        # sort_keys=SortSchemeMapper.map(sort_order, schema),
+    )
+
+    print(f"Getting Glue Table: {namespace}.{table_name}")
+    table_definition = dc.get_table(name=table_name, namespace=namespace)
+    print(f"Retrieved Glue Table: {table_definition}")
+
+    # Read Data from DeltaCAT Table (backed by Iceberg) using Daft
+    daft_dataframe = dc.read_table(table=table_name, namespace=namespace)
+
+    daft_dataframe.where(df["bid"] > 200.0).show()
+    # Expected result:
+    # ╭────────┬─────────┬─────────╮
+    # │ symbol ┆ bid     ┆ ask     │
+    # │ ---    ┆ ---     ┆ ---     │
+    # │ Utf8   ┆ Float64 ┆ Float64 │
+    # ╞════════╪═════════╪═════════╡
+    # │ meta   ┆ 392.03  ┆ 392.09  │
+    # ├╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┤
+    # │ msft   ┆ 403.25  ┆ 403.27  │
+    # ╰────────┴─────────┴─────────╯
+
+    daft_dataframe.select("symbol").show()
+    # Expected result:
+    # ╭────────╮
+    # │ symbol │
+    # │ ---    │
+    # │ Utf8   │
+    # ╞════════╡
+    # │ meta   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ amzn   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ goog   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ msft   │
+    # ╰────────╯
+
+
+if __name__ == "__main__":
+    example_script_args = [
+        (
+            [
+                "--warehouse",
+            ],
+            {
+                "help": "S3 path for Iceberg file storage.",
+                "type": str,
+            },
+        ),
+    ]
+
+    # store any CLI args in the runtime environment
+    store_cli_args_in_os_environ(example_script_args)
+
+    # run the example using os.environ as kwargs
+    run(**os.environ)

deltacat/examples/experimental/iceberg/iceberg_reader.py

@@ -0,0 +1,147 @@
+import os
+import logging
+import deltacat as dc
+
+from deltacat import logs
+from deltacat import IcebergCatalog
+from env import store_cli_args_in_os_environ
+
+from pyiceberg.schema import (
+    Schema,
+    NestedField,
+    DoubleType,
+    StringType,
+    TimestampType,
+    FloatType,
+    StructType,
+)
+from pyiceberg.partitioning import PartitionSpec, PartitionField
+from pyiceberg.transforms import DayTransform, IdentityTransform
+from pyiceberg.table.sorting import SortField, SortOrder
+
+from deltacat.exceptions import TableAlreadyExistsError
+from deltacat.experimental.storage.iceberg.model import (
+    SchemaMapper,
+    PartitionSchemeMapper,
+    SortSchemeMapper,
+)
+from deltacat.env import create_ray_runtime_environment
+
+# initialize the driver logger
+driver_logger = logs.configure_application_logger(logging.getLogger(__name__))
+
+
+def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
+    # create any runtime environment required to run the example
+    runtime_env = create_ray_runtime_environment()
+
+    # Start by initializing DeltaCAT and registering available Catalogs.
+    # Ray will be initialized automatically via `ray.init()`.
+    # Only the `iceberg` data catalog is provided so it will become the default.
+    # If initializing multiple catalogs, use the `default_catalog_name` param
+    # to specify which catalog should be the default.
+    dc.init(
+        catalogs={
+            # the name of the DeltaCAT catalog is "iceberg"
+            "iceberg": dc.Catalog(
+                # Apache Iceberg implementation of deltacat.catalog.interface
+                impl=IcebergCatalog,
+                # kwargs for pyiceberg.catalog.load_catalog start here...
+                # the name of the Iceberg catalog is "example-iceberg-catalog"
+                name="example-iceberg-catalog",
+                # for additional properties see:
+                # https://py.iceberg.apache.org/configuration/
+                properties={
+                    "type": "glue",
+                    "region_name": "us-east-1",
+                    "warehouse": warehouse,
+                },
+            )
+        },
+        # pass the runtime environment into ray.init()
+        ray_init_args={"runtime_env": runtime_env},
+    )
+
+    # define a native Iceberg table schema
+    schema = Schema(
+        NestedField(
+            field_id=1, name="datetime", field_type=TimestampType(), required=True
+        ),
+        NestedField(field_id=2, name="symbol", field_type=StringType(), required=True),
+        NestedField(field_id=3, name="bid", field_type=FloatType(), required=False),
+        NestedField(field_id=4, name="ask", field_type=DoubleType(), required=False),
+        NestedField(
+            field_id=5,
+            name="details",
+            field_type=StructType(
+                NestedField(
+                    field_id=6,
+                    name="created_by",
+                    field_type=StringType(),
+                    required=False,
+                ),
+            ),
+            required=False,
+        ),
+    )
+
+    # define a native Iceberg partition spec
+    partition_spec = PartitionSpec(
+        PartitionField(
+            source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day"
+        )
+    )
+
+    # define a native Iceberg sort order
+    sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform()))
+
+    # create a table named `test_namespace.test_table`
+    # we don't need to specify which catalog to create this table in since
+    # only the "iceberg" catalog is available
+    table_name = "test_table"
+    namespace = "test_namespace"
+    print(f"Creating Glue Table: {namespace}.{table_name}")
+    try:
+        table_definition = dc.create_table(
+            table=table_name,
+            namespace=namespace,
+            schema=SchemaMapper.map(schema),
+            partition_scheme=PartitionSchemeMapper.map(partition_spec, schema),
+            sort_keys=SortSchemeMapper.map(sort_order, schema),
+        )
+        print(f"Created Glue Table: {table_definition}")
+    except TableAlreadyExistsError:
+        print(f"Glue Table `{namespace}.{table_name}` already exists.")
+
+    print(f"Getting Glue Table: {namespace}.{table_name}")
+    table_definition = dc.get_table(table_name, namespace)
+    print(f"Retrieved Glue Table: {table_definition}")
+
+
+if __name__ == "__main__":
+    example_script_args = [
+        (
+            [
+                "--warehouse",
+            ],
+            {
+                "help": "S3 path for Iceberg file storage.",
+                "type": str,
+            },
+        ),
+        (
+            [
+                "--STAGE",
+            ],
+            {
+                "help": "Example runtime environment stage (e.g. dev, alpha, beta, prod).",
+                "type": str,
+            },
+        ),
+    ]
+
+    # store any CLI args in the runtime environment
+    store_cli_args_in_os_environ(example_script_args)
+
+    # run the example using os.environ as kwargs
+    run(**os.environ)

deltacat/examples/hello_world.py

@@ -0,0 +1,29 @@
+import ray
+import deltacat
+import daft
+
+
+def print_package_version_info():
+    print(f"DeltaCAT Version: {deltacat.__version__}")
+    print(f"Ray Version: {ray.__version__}")
+    print(f"Daft Version: {daft.__version__}")
+
+
+@ray.remote
+def hello_worker():
+    print("Hello, Worker!")
+    print_package_version_info()
+
+
+def run():
+    print("Hello, Driver!")
+    print_package_version_info()
+    hello_worker.remote()
+
+
+if __name__ == "__main__":
+    # initialize deltacat
+    deltacat.init()
+
+    # run the example
+    run()