deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/examples/experimental/iceberg/converter/beam/utils/spark.py

@@ -0,0 +1,263 @@
+"""
+Spark SQL utilities for Iceberg table operations.
+
+This module provides Beam DoFn classes that use Spark SQL to work with Iceberg tables,
+"""
+
+import os
+import apache_beam as beam
+from apache_beam import Row
+
+
+class SparkSQLIcebergRead(beam.DoFn):
+    """
+    Custom Beam DoFn that uses Spark SQL to read Iceberg tables.
+    """
+
+    def __init__(
+        self,
+        table_name: str,
+        catalog_uri: str = "http://localhost:8181",
+        warehouse: str = "warehouse/",
+    ):
+        """
+        Initialize the Spark SQL reader.
+
+        Args:
+            table_name: Name of the Iceberg table
+            catalog_uri: URI of the Iceberg REST catalog
+            warehouse: Warehouse path
+        """
+        self.table_name = table_name
+        self.catalog_uri = catalog_uri
+        self.warehouse = warehouse
+        self.spark = None
+
+    def setup(self):
+        """Set up Spark session (called once per worker)."""
+        try:
+            from pyspark.sql import SparkSession
+            import importlib.metadata
+
+            # Get Spark version for dependency resolution
+            try:
+                spark_version = ".".join(
+                    importlib.metadata.version("pyspark").split(".")[:2]
+                )
+            except Exception:
+                spark_version = "3.5"  # Default fallback
+
+            scala_version = "2.12"
+            iceberg_version = "1.6.0"
+
+            print(f"🔧 Setting up Spark session for reading {self.table_name}")
+            print(f"   - Spark version: {spark_version}")
+            print(f"   - Iceberg version: {iceberg_version}")
+
+            # Set Spark packages for Iceberg runtime
+            os.environ["PYSPARK_SUBMIT_ARGS"] = (
+                f"--packages org.apache.iceberg:iceberg-spark-runtime-{spark_version}_{scala_version}:{iceberg_version} "
+                f"pyspark-shell"
+            )
+
+            # Create Spark session with Iceberg REST catalog configuration
+            self.spark = (
+                SparkSession.builder.appName(f"DeltaCAT Read - {self.table_name}")
+                .config("spark.sql.session.timeZone", "UTC")
+                .config(
+                    "spark.serializer", "org.apache.spark.serializer.KryoSerializer"
+                )
+                .config(
+                    "spark.sql.extensions",
+                    "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+                )
+                # Configure REST catalog
+                .config(
+                    "spark.sql.catalog.rest", "org.apache.iceberg.spark.SparkCatalog"
+                )
+                .config("spark.sql.catalog.rest.type", "rest")
+                .config("spark.sql.catalog.rest.uri", self.catalog_uri)
+                .config("spark.sql.catalog.rest.warehouse", self.warehouse)
+                # Set REST as default catalog
+                .config("spark.sql.defaultCatalog", "rest")
+                # Local mode configuration (within Beam workers)
+                .config("spark.master", "local[1]")  # Single thread per worker
+                .config("spark.sql.adaptive.enabled", "true")
+                # Networking binding
+                .config("spark.driver.bindAddress", "127.0.0.1")
+                .config("spark.driver.host", "127.0.0.1")
+                .config("spark.ui.enabled", "false")
+                .config("spark.sql.adaptive.coalescePartitions.enabled", "false")
+                .getOrCreate()
+            )
+
+            print(f"✅ Spark session created successfully")
+
+        except Exception as e:
+            print(f"❌ Failed to set up Spark session: {e}")
+            raise
+
+    def teardown(self):
+        """Clean up Spark session (called once per worker)."""
+        if self.spark:
+            try:
+                self.spark.stop()
+                print("✅ Spark session stopped")
+            except Exception as e:
+                print(f"⚠️ Error stopping Spark session: {e}")
+
+    def process(self, element):
+        """
+        Process element (read from Iceberg table using Spark SQL).
+
+        Args:
+            element: Input element (not used, just triggers the read)
+
+        Yields:
+            Records from the Iceberg table
+        """
+        try:
+            if not self.spark:
+                raise RuntimeError("Spark session not initialized")
+
+            print(f"📖 Reading table {self.table_name} using Spark SQL")
+
+            # Read from Iceberg table using Spark SQL
+            df = self.spark.sql(f"SELECT * FROM {self.table_name}")
+
+            # Collect all records
+            records = df.collect()
+
+            print(f"📊 Successfully read {len(records)} records from {self.table_name}")
+
+            # Convert Spark rows to Beam Row objects and yield
+            for row in records:
+                row_dict = row.asDict()
+                # Convert to Beam Row for consistency with write mode
+                beam_row = Row(**row_dict)
+                yield beam_row
+
+        except Exception as e:
+            print(f"❌ Failed to read from table {self.table_name}: {e}")
+            raise
+
+
+class SparkSQLIcebergRewrite(beam.DoFn):
+    """
+    Custom Beam DoFn that uses Spark SQL to rewrite Iceberg table data files.
+
+    This uses Spark's rewrite_data_files procedure to materialize positional deletes
+    by rewriting data files. The result is a "clean" table without positional deletes.
+    """
+
+    def __init__(self, catalog_uri, warehouse_path, table_name):
+        self.catalog_uri = catalog_uri
+        self.warehouse_path = warehouse_path
+        self.table_name = table_name
+
+    def setup(self):
+        """Initialize Spark session for rewrite operations."""
+        try:
+            from pyspark.sql import SparkSession
+            import importlib.metadata
+
+            print(f"🔧 Setting up Spark session for rewriting {self.table_name}")
+
+            # Detect Spark version for appropriate Iceberg runtime
+            spark_version = importlib.metadata.version("pyspark")
+            major_minor = ".".join(spark_version.split(".")[:2])
+            print(f"   - Spark version: {major_minor}")
+            print(f"   - Iceberg version: 1.6.0")
+
+            # Configure Spark with Iceberg
+            self.spark = (
+                SparkSession.builder.appName("IcebergRewrite")
+                .config(
+                    "spark.jars.packages",
+                    f"org.apache.iceberg:iceberg-spark-runtime-{major_minor}_2.12:1.6.0",
+                )
+                .config(
+                    "spark.sql.extensions",
+                    "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+                )
+                .config(
+                    "spark.sql.catalog.spark_catalog",
+                    "org.apache.iceberg.spark.SparkSessionCatalog",
+                )
+                .config("spark.sql.catalog.spark_catalog.type", "rest")
+                .config("spark.sql.catalog.spark_catalog.uri", self.catalog_uri)
+                .config(
+                    "spark.sql.catalog.spark_catalog.warehouse", self.warehouse_path
+                )
+                .config("spark.driver.bindAddress", "127.0.0.1")
+                .config("spark.driver.host", "127.0.0.1")
+                .config("spark.ui.enabled", "false")
+                .getOrCreate()
+            )
+
+            print("✅ Spark session created successfully")
+
+        except ImportError as e:
+            raise RuntimeError(
+                f"PySpark is required for rewrite mode. Install with: pip install pyspark"
+            ) from e
+        except Exception as e:
+            raise RuntimeError(f"Failed to create Spark session: {e}") from e
+
+    def process(self, element):
+        """Rewrite table data files to materialize positional deletes."""
+        try:
+            print(
+                f"📋 Rewriting table {self.table_name} to materialize positional deletes"
+            )
+
+            # Use Spark's rewrite_data_files procedure with delete_file_threshold=1
+            # This forces rewrite even when there's only 1 positional delete file
+            rewrite_sql = f"""
+            CALL spark_catalog.system.rewrite_data_files(
+                table => '{self.table_name}',
+                options => map('delete-file-threshold', '1')
+            )
+            """
+
+            print(f"🔄 Executing rewrite procedure with delete_file_threshold=1...")
+            print(f"   SQL: {rewrite_sql.strip()}")
+            print(
+                f"   Rationale: Forces rewrite even with single positional delete file"
+            )
+
+            result = self.spark.sql(rewrite_sql)
+
+            # Collect results to see what was rewritten
+            rewrite_result = result.collect()[0]
+            print(f"📊 Rewrite result: {rewrite_result}")
+
+            # Check if we actually rewrote anything
+            if rewrite_result.rewritten_data_files_count > 0:
+                print(
+                    f"✅ Successfully rewrote {rewrite_result.rewritten_data_files_count} data files"
+                )
+                print(
+                    f"   - Added {rewrite_result.added_data_files_count} new data files"
+                )
+                print(f"   - Rewrote {rewrite_result.rewritten_bytes_count} bytes")
+                print(f"   - Positional deletes have been materialized!")
+            else:
+                print(f"⚠️  No files were rewritten (rewritten_data_files_count=0)")
+                print(f"   - This may indicate no positional deletes exist")
+                print(f"   - Or the table may already be in optimal state")
+
+            yield f"Rewrite completed for {self.table_name}"
+
+        except Exception as e:
+            print(f"❌ Error during rewrite: {e}")
+            import traceback
+
+            traceback.print_exc()
+            yield f"Rewrite failed for {self.table_name}: {e}"
+
+    def teardown(self):
+        """Clean up Spark session."""
+        if hasattr(self, "spark"):
+            print("✅ Spark session stopped")
+            self.spark.stop()

deltacat/examples/{iceberg → experimental/iceberg}/iceberg_bucket_writer.py

@@ -1,14 +1,16 @@
 import os
 import logging
 
+import uuid
 import daft
+from pyiceberg.catalog import CatalogType
+
 import deltacat as dc
 
 from deltacat import logs
 from deltacat import IcebergCatalog
-from deltacat.examples.common.fixtures import (
-    store_cli_args_in_os_environ,
-)
+from deltacat.experimental.catalog.iceberg import IcebergCatalogConfig
+from env import store_cli_args_in_os_environ
 
 from pyiceberg.schema import (
     Schema,
@@ -19,7 +21,7 @@ from pyiceberg.schema import (
 from pyiceberg.partitioning import PartitionSpec, PartitionField
 from pyiceberg.transforms import BucketTransform
 
-from deltacat.storage.iceberg.model import (
+from deltacat.experimental.storage.iceberg.model import (
     SchemaMapper,
     PartitionSchemeMapper,
 )
@@ -30,6 +32,24 @@ driver_logger = logs.configure_application_logger(logging.getLogger(__name__))
 
 
 def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
+    """
+    This is an e2e example that
+        1. creates a DeltaCAT Table (backed by an Iceberg Table) in Glue
+        2. writes data into the DeltaCAT Table
+        3. reads data from the DeltaCAT Table using Daft
+
+    To run the script:
+        1. prepare an AWS Account
+            1. prepare a S3 location where the data will be written to, which will be used in Step 3.
+            2. prepare an IAM Role that has access to the S3 location and Glue
+        2. retrieve the IAM Role AWS Credential and cache locally in ~/.aws/credentials
+        3. run below command to execute the example
+        ```
+        make venv && source venv/bin/activate
+        python -m deltacat.examples.iceberg.iceberg_bucket_writer --warehouse=s3://<YOUR_S3_LOCATION>
+        ```
+
+    """
     # create any runtime environment required to run the example
     runtime_env = create_ray_runtime_environment()
 
@@ -38,6 +58,7 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
     # Only the `iceberg` data catalog is provided so it will become the default.
     # If initializing multiple catalogs, use the `default_catalog_name` param
     # to specify which catalog should be the default.
+
     dc.init(
         catalogs={
             # the name of the DeltaCAT catalog is "iceberg"
@@ -49,11 +70,13 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
                 name="example-iceberg-catalog",
                 # for additional properties see:
                 # https://py.iceberg.apache.org/configuration/
-                properties={
-                    "type": "glue",
-                    "region_name": "us-east-1",
-                    "warehouse": warehouse,
-                },
+                config=IcebergCatalogConfig(
+                    type=CatalogType.GLUE,
+                    properties={
+                        "warehouse": warehouse,
+                        "region_name": "us-east-1",
+                    },
+                ),
             )
         },
         # pass the runtime environment into ray.init()
@@ -89,10 +112,10 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
         }
     )
 
-    # write to a table named `test_namespace.test_table_bucketed`
+    # write to a table named `test_namespace.test_table_bucketed-<SUFFIX>`
     # we don't need to specify which catalog to create this table in since
     # only the "iceberg" catalog is available
-    table_name = "test_table_bucketed"
+    table_name = f"test_table_bucketed-{uuid.uuid4().hex[:8]}"
     namespace = "test_namespace"
     print(f"Creating Glue Table: {namespace}.{table_name}")
     dc.write_to_table(
@@ -106,9 +129,40 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
     )
 
     print(f"Getting Glue Table: {namespace}.{table_name}")
-    table_definition = dc.get_table(table_name, namespace)
+    table_definition = dc.get_table(name=table_name, namespace=namespace)
     print(f"Retrieved Glue Table: {table_definition}")
 
+    # Read Data from DeltaCAT Table (backed by Iceberg) using Daft
+    daft_dataframe = dc.read_table(table=table_name, namespace=namespace)
+
+    daft_dataframe.where(df["bid"] > 200.0).show()
+    # Expected result:
+    # ╭────────┬─────────┬─────────╮
+    # │ symbol ┆ bid     ┆ ask     │
+    # │ ---    ┆ ---     ┆ ---     │
+    # │ Utf8   ┆ Float64 ┆ Float64 │
+    # ╞════════╪═════════╪═════════╡
+    # │ meta   ┆ 392.03  ┆ 392.09  │
+    # ├╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┤
+    # │ msft   ┆ 403.25  ┆ 403.27  │
+    # ╰────────┴─────────┴─────────╯
+
+    daft_dataframe.select("symbol").show()
+    # Expected result:
+    # ╭────────╮
+    # │ symbol │
+    # │ ---    │
+    # │ Utf8   │
+    # ╞════════╡
+    # │ meta   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ amzn   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ goog   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ msft   │
+    # ╰────────╯
+
 
 if __name__ == "__main__":
     example_script_args = [
@@ -121,15 +175,6 @@ if __name__ == "__main__":
                 "type": str,
             },
         ),
-        (
-            [
-                "--STAGE",
-            ],
-            {
-                "help": "Example runtime environment stage (e.g. dev, alpha, beta, prod).",
-                "type": str,
-            },
-        ),
     ]
 
     # store any CLI args in the runtime environment

deltacat/examples/{iceberg → experimental/iceberg}/iceberg_reader.py

@@ -4,9 +4,7 @@ import deltacat as dc
 
 from deltacat import logs
 from deltacat import IcebergCatalog
-from deltacat.examples.common.fixtures import (
-    store_cli_args_in_os_environ,
-)
+from env import store_cli_args_in_os_environ
 
 from pyiceberg.schema import (
     Schema,
@@ -22,7 +20,7 @@ from pyiceberg.transforms import DayTransform, IdentityTransform
 from pyiceberg.table.sorting import SortField, SortOrder
 
 from deltacat.exceptions import TableAlreadyExistsError
-from deltacat.storage.iceberg.model import (
+from deltacat.experimental.storage.iceberg.model import (
     SchemaMapper,
     PartitionSchemeMapper,
     SortSchemeMapper,

deltacat/examples/hello_world.py

@@ -1,12 +1,10 @@
 import ray
 import deltacat
 import daft
-import pyiceberg
 
 
 def print_package_version_info():
     print(f"DeltaCAT Version: {deltacat.__version__}")
-    print(f"PyIceberg Version: {pyiceberg.__version__}")
     print(f"Ray Version: {ray.__version__}")
     print(f"Daft Version: {daft.__version__}")
 
@@ -24,4 +22,8 @@ def run():
 
 
 if __name__ == "__main__":
+    # initialize deltacat
+    deltacat.init()
+
+    # run the example
     run()

deltacat/examples/indexer/indexer.py

@@ -0,0 +1,163 @@
+import argparse
+
+from datetime import datetime
+
+import ray
+
+import deltacat
+import daft
+import pyarrow as pa
+import pandas as pd
+import polars as pl
+import numpy as np
+
+from deltacat import DeltaCatUrl
+
+
+def print_package_version_info() -> None:
+    print(f"DeltaCAT Version: {deltacat.__version__}")
+    print(f"Ray Version: {ray.__version__}")
+    print(f"Daft Version: {daft.__version__}")
+    print(f"NumPy Version: {np.__version__}")
+    print(f"PyArrow Version: {pa.__version__}")
+    print(f"Polars Version: {pl.__version__}")
+    print(f"Pandas Version: {pd.__version__}")
+
+
+def json_path_to_regex(path: str):
+    if not path:
+        raise ValueError("Path cannot be empty")
+    parts = path.split("/")
+    leaf_key = parts.pop()
+    regex = r""
+    for part in parts:
+        if part.strip():  # discard leading and/or redundant separators
+            regex += rf'"{part}"\s*:\s*[{{\[].*?'
+    regex += rf'"{leaf_key}"\s*:\s*"(?<{leaf_key}>.*?)"'
+    return regex
+
+
+def run(
+    source: str,
+    dest: str,
+) -> None:
+    # print package version info
+    print_package_version_info()
+
+    # run a synchronous copy from the source to the destination
+    deltacat.copy(
+        DeltaCatUrl(source),
+        DeltaCatUrl(dest),
+        # reader arguments to pass to the default reader (polars)
+        # for the given text-based datasource, it accepts the same
+        # arguments as polars.read_csv except for `source`, `n_threads`
+        # `new_columns`, `separator`, `has_header`, `quote_char`, and
+        # `infer_schema`.
+        reader_args={
+            "low_memory": True,  # try to use less memory (++stability, --perf)
+            "batch_size": 1024,  # text line count read into a buffer at once
+            "use_pyarrow": True,  # use the native pyarrow reader
+        },
+        # writer arguments to pass to the default writer (polars)
+        # for the given parquet-based datasink, it generally accepts the same
+        # arguments as polars.DataFrame.write_{dest-type} except for `file`
+        writer_args={
+            "compression": "lz4",  # faster compression & decompression
+            # "compression": "zstd",  # better compression ratio
+            # "compression": "snappy",  # compatible w/ older Parquet readers
+        },
+        # Transforms to run against the default polars dataframe read.
+        # By default, each transform takes a polars dataframe `df` as input
+        # and produces a polars dataframe as output. All transforms listed
+        # are run in order (i.e., the dataframe output from transform[0]
+        # is the dataframe input to transform[1]).
+        #
+        # See:
+        # https://docs.pola.rs/api/python/stable/reference/dataframe/index.html
+        # https://docs.pola.rs/api/python/stable/reference/expressions/index.html
+        transforms=[
+            lambda df, src: df.rename(
+                {"text": "utf8_body"},
+            ),
+            lambda df, src: df.with_columns(
+                pl.col("utf8_body").hash().alias("utf8_body_hash"),
+                pl.lit(datetime.utcnow()).dt.datetime().alias("processing_time"),
+                pl.lit(src.url_path).alias("source_file_path"),
+            ),
+        ],
+    )
+
+
+if __name__ == "__main__":
+    """
+    Example 1: Run this script locally using Ray:
+    $ python indexer.py \
+    $   --source 'text+s3://openalex-mag-format/data_dump_v1/2022-07-08/nlp/PaperAbstractsInvertedIndex.txt_part31' \
+    $   --dest 'parquet+s3://deltacat-example-output/openalex/PaperAbstractsInvertedIndex.part31.parquet'
+
+    Example 2: Submit this script as a local Ray job using a local job client:
+    >>> from deltacat import local_job_client
+    >>> client = local_job_client()
+    >>> # read the source file as line-delimited text
+    >>> src = "text+s3://openalex-mag-format/data_dump_v1/2022-07-08/nlp/PaperAbstractsInvertedIndex.txt_part31"
+    >>> # write to the destination file using the default DeltaCAT Parquet writer (i.e., polars.DataFrame.write_parquet)
+    >>> dst = "parquet+s3://deltacat-example-output/openalex/PaperAbstractsInvertedIndex.part31.parquet"
+    >>> try:
+    >>>   job_run_result = client.run_job(
+    >>>       # Entrypoint shell command to run the indexer job
+    >>>       entrypoint=f"python indexer.py --source '{src}' --dest '{dst}'",
+    >>>       # Path to the local directory that contains the indexer.py file
+    >>>       runtime_env={"working_dir": "./deltacat/examples/indexer.py"},
+    >>>   )
+    >>>   print(f"Job ID {job_run_result.job_id} terminal state: {job_run_result.job_status}")
+    >>>   print(f"Job ID {job_run_result.job_id} logs: ")
+    >>>   print(job_run_result.job_logs)
+    >>> except RuntimeError as e:
+    >>>     print(f"Job Run Failed: {e}")
+    >>> except TimeoutError as e:
+    >>>     print(f"Job Run Timed Out: {e}")
+
+    Example 3: Submit this script as a remote Ray job using a remote job client:
+    >>> from deltacat import job_client
+    >>> # use `deltacat.yaml` from the current working directory as the ray cluster launcher config file
+    >>> # automatically launches the cluster if it doesn't exist or has died
+    >>> # automatically forwards the ray cluster's dashboard for viewing in a web browser @ http://localhost:8265
+    >>> client = job_client()
+    >>> # ... follow the same steps as above to submit a synchronous indexer job ...
+    >>>
+    >>> # OR use an explicit cluster launcher config file path
+    >>> client = job_client("/Users/pdames/workspace/deltacat.yaml")
+    >>> # ... follow the same steps as above to submit a synchronous indexer job ...
+    """
+    script_args = [
+        (
+            [
+                "--source",
+            ],
+            {
+                "help": "Source DeltaCAT URL to index.",
+                "type": str,
+            },
+        ),
+        (
+            [
+                "--dest",
+            ],
+            {
+                "help": "Destination DeltaCAT URL to index.",
+                "type": str,
+            },
+        ),
+    ]
+    # parse CLI input arguments
+    parser = argparse.ArgumentParser()
+    for args, kwargs in script_args:
+        parser.add_argument(*args, **kwargs)
+    args = parser.parse_args()
+    print(f"Command Line Arguments: {args}")
+
+    # initialize deltacat
+    deltacat.init()
+
+    # run the example using the parsed arguments
+    run(**vars(args))