deltacat 1.1.35__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/compute/converter/utils/converter_session_utils.py

@@ -0,0 +1,109 @@
+from collections import defaultdict
+import logging
+from deltacat import logs
+from deltacat.compute.converter.model.convert_input_files import ConvertInputFiles
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def check_data_files_sequence_number(data_files_list, equality_delete_files_list):
+    # Sort by file sequence number
+    data_files_list.sort(key=lambda file_tuple: file_tuple[0])
+    equality_delete_files_list.sort(key=lambda file_tuple: file_tuple[0])
+
+    equality_delete_files = []
+    result_data_file = []
+
+    # Pointer for list data_file
+    data_file_pointer = 0
+
+    # Loop through each value in equality_delete_file
+    for equality_file_tuple in equality_delete_files_list:
+        # Find all values in data_file that are smaller than val_equality
+        valid_values = []
+
+        # Move data_file_pointer to the first value in data_file that is smaller than val_equality
+        while (
+            data_file_pointer < len(data_files_list)
+            and data_files_list[data_file_pointer][0] < equality_file_tuple[0]
+        ):
+            valid_values.append(data_files_list[data_file_pointer])
+            data_file_pointer += 1
+            equality_delete_files.append(equality_file_tuple)
+
+        # Append the value from equality_delete_file and the corresponding valid values from data_file
+        if valid_values:
+            result_data_file.append(valid_values)
+
+    result_equality_delete_file = append_larger_sequence_number_data_files(
+        equality_delete_files
+    )
+
+    return result_equality_delete_file, result_data_file
+
+
+def append_larger_sequence_number_data_files(data_files_list):
+    result = []
+    # Iterate over the input list
+    for i in range(len(data_files_list)):
+        sublist = data_files_list[i:]
+        sublist_file_list = []
+        for file in sublist:
+            sublist_file_list.append(file)
+        result.append(sublist_file_list)
+    return result
+
+
+def construct_iceberg_table_prefix(
+    iceberg_warehouse_bucket_name, table_name, iceberg_namespace
+):
+    return f"{iceberg_warehouse_bucket_name}/{iceberg_namespace}/{table_name}/data"
+
+
+def partition_value_record_to_partition_value_string(partition):
+    # Get string representation of partition value out of Record[partition_value]
+    partition_value_str = partition.__repr__().split("[", 1)[1].split("]")[0]
+    return partition_value_str
+
+
+def group_all_files_to_each_bucket(
+    data_file_dict, equality_delete_dict, pos_delete_dict
+):
+    convert_input_files_for_all_buckets = []
+    files_for_each_bucket_for_deletes = defaultdict(tuple)
+    if equality_delete_dict:
+        for partition_value, equality_delete_file_list in equality_delete_dict.items():
+            (
+                result_equality_delete_file,
+                result_data_file,
+            ) = check_data_files_sequence_number(
+                data_files_list=data_file_dict[partition_value],
+                equality_delete_files_list=equality_delete_dict[partition_value],
+            )
+            files_for_each_bucket_for_deletes[partition_value] = (
+                result_data_file,
+                result_equality_delete_file,
+                [],
+            )
+            if partition_value not in data_file_dict:
+                convert_input_file = ConvertInputFiles.of(
+                    partition_value=partition_value,
+                    applicable_data_files=result_data_file,
+                    applicable_equalitu_delete_files=result_equality_delete_file,
+                )
+                convert_input_files_for_all_buckets.append(convert_input_file)
+
+    for partition_value, all_data_files_for_each_bucket in data_file_dict.items():
+        convert_input_file = ConvertInputFiles.of(
+            partition_value=partition_value,
+            all_data_files_for_dedupe=all_data_files_for_each_bucket,
+        )
+        if partition_value in files_for_each_bucket_for_deletes:
+            convert_input_file.applicable_data_files = (
+                files_for_each_bucket_for_deletes[partition_value][0]
+            )
+            convert_input_file.applicable_delete_files = (
+                files_for_each_bucket_for_deletes[partition_value][1]
+            )
+        convert_input_files_for_all_buckets.append(convert_input_file)
+    return convert_input_files_for_all_buckets

deltacat/compute/converter/utils/iceberg_columns.py

@@ -0,0 +1,82 @@
+import pyarrow as pa
+from typing import Union
+import numpy as np
+
+# Refer to: https://iceberg.apache.org/spec/#reserved-field-ids for reserved field ids
+ICEBERG_RESERVED_FIELD_ID_FOR_FILE_PATH_COLUMN = 2147483546
+
+# Refer to: https://iceberg.apache.org/spec/#reserved-field-ids for reserved field ids
+ICEBERG_RESERVED_FIELD_ID_FOR_POS_COLUMN = 2147483545
+
+
+def _get_iceberg_col_name(suffix):
+    return suffix
+
+
+_ORDERED_RECORD_IDX_COLUMN_NAME = _get_iceberg_col_name("pos")
+_ORDERED_RECORD_IDX_COLUMN_TYPE = pa.int64()
+_ORDERED_RECORD_IDX_FIELD_METADATA = {
+    b"PARQUET:field_id": f"{ICEBERG_RESERVED_FIELD_ID_FOR_POS_COLUMN}"
+}
+_ORDERED_RECORD_IDX_COLUMN_FIELD = pa.field(
+    _ORDERED_RECORD_IDX_COLUMN_NAME,
+    _ORDERED_RECORD_IDX_COLUMN_TYPE,
+    metadata=_ORDERED_RECORD_IDX_FIELD_METADATA,
+    nullable=False,
+)
+
+
+def get_record_index_column_array(obj) -> Union[pa.Array, pa.ChunkedArray]:
+    return pa.array(
+        obj,
+        _ORDERED_RECORD_IDX_COLUMN_TYPE,
+    )
+
+
+def append_record_idx_col(table: pa.Table, ordered_record_indices) -> pa.Table:
+
+    table = table.append_column(
+        _ORDERED_RECORD_IDX_COLUMN_FIELD,
+        get_record_index_column_array(ordered_record_indices),
+    )
+    return table
+
+
+_FILE_PATH_COLUMN_NAME = _get_iceberg_col_name("file_path")
+_FILE_PATH_COLUMN_TYPE = pa.string()
+_FILE_PATH_FIELD_METADATA = {
+    b"PARQUET:field_id": f"{ICEBERG_RESERVED_FIELD_ID_FOR_FILE_PATH_COLUMN}"
+}
+_FILE_PATH_COLUMN_FIELD = pa.field(
+    _FILE_PATH_COLUMN_NAME,
+    _FILE_PATH_COLUMN_TYPE,
+    metadata=_FILE_PATH_FIELD_METADATA,
+    nullable=False,
+)
+
+
+def append_file_path_column(table: pa.Table, file_path: str):
+    table = table.append_column(
+        _FILE_PATH_COLUMN_FIELD,
+        pa.array(np.repeat(file_path, len(table)), _FILE_PATH_COLUMN_TYPE),
+    )
+    return table
+
+
+_GLOBAL_RECORD_IDX_COLUMN_NAME = _get_iceberg_col_name("global_record_index")
+_GLOBAL_RECORD_IDX_COLUMN_TYPE = pa.int64()
+_GLOBAL_RECORD_IDX_COLUMN_FIELD = pa.field(
+    _GLOBAL_RECORD_IDX_COLUMN_NAME,
+    _GLOBAL_RECORD_IDX_COLUMN_TYPE,
+)
+
+
+def append_global_record_idx_column(
+    table: pa.Table, ordered_record_indices
+) -> pa.Table:
+
+    table = table.append_column(
+        _GLOBAL_RECORD_IDX_COLUMN_NAME,
+        pa.array(ordered_record_indices, _GLOBAL_RECORD_IDX_COLUMN_TYPE),
+    )
+    return table

deltacat/compute/converter/utils/io.py

@@ -0,0 +1,43 @@
+import deltacat.compute.converter.utils.iceberg_columns as sc
+import daft
+
+
+def download_data_table_and_append_iceberg_columns(
+    file, columns_to_download, additional_columns_to_append, sequence_number
+):
+    # TODO; add S3 client kwargs
+    table = download_parquet_with_daft_hash_applied(
+        identify_columns=columns_to_download, file=file, s3_client_kwargs={}
+    )
+    if sc._FILE_PATH_COLUMN_NAME in additional_columns_to_append:
+        table = sc.append_file_path_column(table, file.file_path)
+    if sc._ORDERED_RECORD_IDX_COLUMN_NAME in additional_columns_to_append:
+        record_idx_iterator = iter(range(len(table)))
+        table = sc.append_record_idx_col(table, record_idx_iterator)
+    return table
+
+
+def download_parquet_with_daft_hash_applied(
+    identify_columns, file, s3_client_kwargs, **kwargs
+):
+    from daft import TimeUnit
+
+    # TODO: Add correct read kwargs as in:
+    #  https://github.com/ray-project/deltacat/blob/383855a4044e4dfe03cf36d7738359d512a517b4/deltacat/utils/daft.py#L97
+
+    coerce_int96_timestamp_unit = TimeUnit.from_str(
+        kwargs.get("coerce_int96_timestamp_unit", "ms")
+    )
+
+    from deltacat.utils.daft import _get_s3_io_config
+
+    # TODO: Use Daft SHA1 hash instead to minimize probably of data corruption
+    io_config = _get_s3_io_config(s3_client_kwargs=s3_client_kwargs)
+    df = daft.read_parquet(
+        path=file.file_path,
+        io_config=io_config,
+        coerce_int96_timestamp_unit=coerce_int96_timestamp_unit,
+    )
+    df = df.select(daft.col(identify_columns[0]).hash())
+    arrow_table = df.to_arrow()
+    return arrow_table

deltacat/compute/converter/utils/s3u.py

@@ -0,0 +1,133 @@
+from tenacity import (
+    Retrying,
+    retry_if_exception_type,
+    stop_after_delay,
+    wait_random_exponential,
+)
+from typing import Union
+from deltacat.aws.s3u import CapturedBlockWritePaths, UuidBlockWritePathProvider
+from deltacat.types.tables import (
+    get_table_writer,
+    get_table_length,
+    TABLE_CLASS_TO_SLICER_FUNC,
+)
+from typing import Optional, Dict, Any, List
+from deltacat.exceptions import RetryableError
+from deltacat.storage import (
+    DistributedDataset,
+    LocalTable,
+)
+from deltacat.types.media import (
+    ContentEncoding,
+    ContentType,
+)
+from deltacat.aws.s3u import UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY
+import s3fs
+
+
+def get_credential():
+    import boto3
+
+    boto3_session = boto3.Session()
+    credentials = boto3_session.get_credentials()
+    return credentials
+
+
+def get_s3_file_system(content_type):
+    token_holder = get_credential()
+    content_encoding = ContentEncoding.IDENTITY
+
+    s3_file_system = s3fs.S3FileSystem(
+        key=token_holder.access_key,
+        secret=token_holder.secret_key,
+        token=token_holder.token,
+        s3_additional_kwargs={
+            "ServerSideEncryption": "aws:kms",
+            # TODO: Get tagging from table properties
+            "ContentType": content_type.value,
+            "ContentEncoding": content_encoding.value,
+        },
+    )
+    return s3_file_system
+
+
+def upload_table_with_retry(
+    table: Union[LocalTable, DistributedDataset],
+    s3_url_prefix: str,
+    s3_table_writer_kwargs: Optional[Dict[str, Any]],
+    content_type: ContentType = ContentType.PARQUET,
+    max_records_per_file: Optional[int] = 4000000,
+    s3_file_system=None,
+    **s3_client_kwargs,
+) -> List[str]:
+    """
+    Writes the given table to 1 or more S3 files and return Redshift
+    manifest entries describing the uploaded files.
+    """
+    retrying = Retrying(
+        wait=wait_random_exponential(multiplier=1, max=60),
+        stop=stop_after_delay(UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY),
+        retry=retry_if_exception_type(RetryableError),
+    )
+
+    if s3_table_writer_kwargs is None:
+        s3_table_writer_kwargs = {}
+
+    if not s3_file_system:
+        s3_file_system = get_s3_file_system(content_type=content_type)
+    capture_object = CapturedBlockWritePaths()
+    block_write_path_provider = UuidBlockWritePathProvider(
+        capture_object=capture_object
+    )
+    s3_table_writer_func = get_table_writer(table)
+    table_record_count = get_table_length(table)
+    if max_records_per_file is None or not table_record_count:
+        retrying(
+            fn=upload_table,
+            table_slices=table,
+            s3_base_url=f"{s3_url_prefix}",
+            s3_file_system=s3_file_system,
+            s3_table_writer_func=s3_table_writer_func,
+            s3_table_writer_kwargs=s3_table_writer_kwargs,
+            block_write_path_provider=block_write_path_provider,
+            content_type=content_type,
+            **s3_client_kwargs,
+        )
+    else:
+        table_slicer_func = TABLE_CLASS_TO_SLICER_FUNC.get(type(table))
+        table_slices = table_slicer_func(table, max_records_per_file)
+        for table_slice in table_slices:
+            retrying(
+                fn=upload_table,
+                table_slices=table_slice,
+                s3_base_url=f"{s3_url_prefix}",
+                s3_file_system=s3_file_system,
+                s3_table_writer_func=s3_table_writer_func,
+                s3_table_writer_kwargs=s3_table_writer_kwargs,
+                block_write_path_provider=block_write_path_provider,
+                content_type=content_type,
+                **s3_client_kwargs,
+            )
+    del block_write_path_provider
+    write_paths = capture_object.write_paths()
+    return write_paths
+
+
+def upload_table(
+    table_slices,
+    s3_base_url,
+    s3_file_system,
+    s3_table_writer_func,
+    block_write_path_provider,
+    content_type,
+    s3_table_writer_kwargs,
+):
+    s3_table_writer_func(
+        table_slices,
+        s3_base_url,
+        s3_file_system,
+        block_write_path_provider,
+        content_type.value,
+        **s3_table_writer_kwargs,
+    )
+    # TODO: Add a proper fix for block_refs and write_paths not persisting in Ray actors

deltacat/compute/resource_estimation/delta.py

@@ -93,29 +93,11 @@ def _estimate_resources_required_to_process_delta_using_type_params(
                 on_disk_size_bytes=delta.meta.content_length,
             ),
         )
-    file_reader_kwargs_provider = kwargs.get(
-        "file_reader_kwargs_provider"
-    ) or deltacat_storage_kwargs.get("file_reader_kwargs_provider")
-
-    """
-    NOTE: The file_reader_kwargs_provider parameter can be passed in two ways:
-    1. Nested within deltacat_storage_kwargs during resource estimation
-    2. As a top-level attribute of CompactPartitionsParams during compaction
-
-    This creates an inconsistent parameter path between resource estimation and compaction flows.
-    As a long-term solution, this should be unified to use a single consistent path (either always
-    nested in deltacat_storage_kwargs or always as a top-level parameter).
-
-    For now, this implementation handles the resource estimation case by:
-    1. First checking for file_reader_kwargs_provider as a direct kwarg
-    2. Falling back to deltacat_storage_kwargs if not found
-    This approach maintains backward compatibility by not modifying the DELTA_RESOURCE_ESTIMATION_FUNCTIONS signatures.
-    """
+
     appended = append_content_type_params(
         delta=delta,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,
-        file_reader_kwargs_provider=file_reader_kwargs_provider,
     )
 
     if not appended:

deltacat/constants.py

@@ -1,4 +1,8 @@
-from deltacat.utils.common import env_string
+from __future__ import annotations
+
+
+from deltacat.utils.common import env_string, env_bool
+import os
 
 # Environment variables
 DELTACAT_SYS_LOG_LEVEL = env_string("DELTACAT_SYS_LOG_LEVEL", "DEBUG")
@@ -30,6 +34,26 @@ DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME = env_string(
 )
 # A json context which will be logged along with other context args.
 DELTACAT_LOGGER_CONTEXT = env_string("DELTACAT_LOGGER_CONTEXT", None)
+DELTACAT_LOGGER_USE_SINGLE_HANDLER = env_bool(
+    "DELTACAT_LOGGER_USE_SINGLE_HANDLER",
+    False,
+)
+DELTACAT_ROOT = env_string(
+    "DELTACAT_ROOT",
+    os.path.join(os.getcwd(), ".deltacat"),
+)
+
+# CLI Args
+METAFILE_FORMAT_KEY = "METAFILE_FORMAT"
+METAFILE_FORMAT_JSON = "json"
+METAFILE_FORMAT_MSGPACK = "msgpack"
+METAFILE_FORMAT = env_string(METAFILE_FORMAT_KEY, METAFILE_FORMAT_MSGPACK)
+SUPPORTED_METAFILE_FORMATS = [METAFILE_FORMAT_JSON, METAFILE_FORMAT_MSGPACK]
+METAFILE_EXT = {
+    "json": ".json",
+    "msgpack": ".mpk",
+}[METAFILE_FORMAT]
+
 
 # Byte Units
 BYTES_PER_KIBIBYTE = 2**10
@@ -41,6 +65,11 @@ BYTES_PER_PEBIBYTE = 2**50
 SIGNED_INT64_MIN_VALUE = -(2**63)
 SIGNED_INT64_MAX_VALUE = 2**63 - 1
 
+# Time Units
+NANOS_PER_SEC = 1_000_000_000
+MICROS_PER_SEC = 1_000_000
+MILLIS_PER_SEC = 1000
+
 # Inflation multiplier from snappy-compressed parquet to pyarrow.
 # This should be kept larger than actual average inflation multipliers.
 # Note that this is a very rough guess since actual observed pyarrow
@@ -58,3 +87,20 @@ MEMORY_TO_HASH_BUCKET_COUNT_RATIO = 0.0512 * BYTES_PER_TEBIBYTE
 
 # The number of bytes allocated to null values in string physical type in parquet
 NULL_SIZE_BYTES = 4
+
+# Metastore Constants
+REVISION_DIR_NAME: str = "rev"
+TXN_DIR_NAME: str = "txn"
+RUNNING_TXN_DIR_NAME: str = "running"
+FAILED_TXN_DIR_NAME: str = "failed"
+SUCCESS_TXN_DIR_NAME: str = "success"
+TXN_PART_SEPARATOR = "_"
+# Storage interface defaults
+# These defaults should be applied in catalog interface implementations
+# Storage interface implementations should be agnostic to defaults and require full information
+DEFAULT_CATALOG = "DEFAULT"
+DEFAULT_NAMESPACE = "DEFAULT"
+DEFAULT_TABLE_VERSION = "1"
+DEFAULT_STREAM_ID = "stream"
+DEFAULT_PARTITION_ID = "partition"
+DEFAULT_PARTITION_VALUES = ["default"]

deltacat/env.py

@@ -0,0 +1,51 @@
+import os
+import logging
+from typing import Dict, Any
+
+from deltacat import logs
+
+from deltacat.constants import (
+    DELTACAT_APP_LOG_LEVEL,
+    DELTACAT_SYS_LOG_LEVEL,
+    DELTACAT_APP_LOG_DIR,
+    DELTACAT_SYS_LOG_DIR,
+    DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
+    DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
+    DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
+    DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
+    DELTACAT_LOGGER_USE_SINGLE_HANDLER,
+)
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def create_ray_runtime_environment() -> Dict[str, Any]:
+    # log the system environment for debugging
+    logger.debug(f"System Environment: {os.environ}")
+
+    # read the stage (e.g. alpha, beta, dev, etc.) from system environment vars
+    stage = os.environ.get("STAGE")
+    logger.debug(f"Runtime Environment Stage: {stage}")
+    runtime_environment = None
+    if stage:
+        worker_env_vars = {
+            # forward the STAGE environment variable to workers
+            "STAGE": stage,
+            # forward deltacat logging environment variables to workers
+            "DELTACAT_APP_LOG_LEVEL": DELTACAT_APP_LOG_LEVEL,
+            "DELTACAT_SYS_LOG_LEVEL": DELTACAT_SYS_LOG_LEVEL,
+            "DELTACAT_APP_LOG_DIR": DELTACAT_APP_LOG_DIR,
+            "DELTACAT_SYS_LOG_DIR": DELTACAT_SYS_LOG_DIR,
+            "DELTACAT_APP_INFO_LOG_BASE_FILE_NAME": DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
+            "DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME": DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
+            "DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME": DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
+            "DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME": DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
+            "DELTACAT_LOGGER_USE_SINGLE_HANDLER": str(
+                DELTACAT_LOGGER_USE_SINGLE_HANDLER
+            ),
+        }
+        # setup runtime environment from system environment variables:
+        runtime_environment = {
+            "env_vars": worker_env_vars,
+        }
+    return runtime_environment

deltacat/examples/basic_logging.py

@@ -0,0 +1,101 @@
+import os
+import ray
+import logging
+
+from deltacat import logs
+from deltacat.constants import DELTACAT_APP_LOG_DIR, DELTACAT_SYS_LOG_DIR
+from deltacat.examples.common.fixtures import (
+    store_cli_args_in_os_environ,
+)
+from deltacat.env import create_ray_runtime_environment
+
+# initialize the driver logger
+driver_logger = logs.configure_application_logger(logging.getLogger(__name__))
+
+
+@ray.remote
+def logging_worker(var1, var2):
+    # for AWS Glue, worker loggers must be initialized within the worker process
+    worker_logger = logs.configure_application_logger(logging.getLogger(__name__))
+
+    log_line_1 = f"Worker System Environment: {os.environ}"
+    print(
+        f"Writing DEBUG log line from Worker to {DELTACAT_APP_LOG_DIR}: '{log_line_1}'"
+    )
+    worker_logger.debug(log_line_1)
+
+    log_line_2 = f"Worker Variable 1: {var1}"
+    print(
+        f"Writing INFO log line from Worker to {DELTACAT_APP_LOG_DIR}: '{log_line_2}'"
+    )
+    worker_logger.info(log_line_2)
+
+    log_line_3 = f"Worker Variable 2: {var2}"
+    print(
+        f"Writing INFO log line from Worker to {DELTACAT_APP_LOG_DIR}: '{log_line_3}'"
+    )
+    worker_logger.info(log_line_3)
+
+
+def run(var1="default1", var2="default2", **kwargs):
+    log_line_1 = f"Driver Variable 1: {var1}"
+    print(
+        f"Writing INFO log line from Driver to {DELTACAT_APP_LOG_DIR}: '{log_line_1}'"
+    )
+    driver_logger.info(log_line_1)
+
+    log_line_2 = f"Driver Variable 2: {var2}"
+    print(
+        f"Writing INFO log line from Driver to {DELTACAT_APP_LOG_DIR}: '{log_line_2}'"
+    )
+    driver_logger.info(log_line_2)
+
+    print("Starting worker...")
+    ray.get(logging_worker.remote(var1, var2))
+    print(
+        f"The driver is shutting down. Additional DeltaCAT system logs have been written to {DELTACAT_SYS_LOG_DIR}"
+    )
+
+
+if __name__ == "__main__":
+    example_script_args = [
+        (
+            [
+                "--var1",
+            ],
+            {
+                "help": "First argument to log.",
+                "type": str,
+            },
+        ),
+        (
+            [
+                "--var2",
+            ],
+            {
+                "help": "Second argument to log.",
+                "type": str,
+            },
+        ),
+        (
+            [
+                "--STAGE",
+            ],
+            {
+                "help": "Example runtime environment stage (e.g. dev, alpha, beta, prod).",
+                "type": str,
+            },
+        ),
+    ]
+
+    # store any CLI args in the runtime environment
+    store_cli_args_in_os_environ(example_script_args)
+
+    # create any runtime environment required to run the example
+    runtime_env = create_ray_runtime_environment()
+
+    # initialize ray
+    ray.init(runtime_env=runtime_env)
+
+    # run the example using os.environ as kwargs
+    run(**os.environ)

deltacat/examples/common/fixtures.py

@@ -0,0 +1,15 @@
+import os
+import logging
+import argparse
+from deltacat import logs
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def store_cli_args_in_os_environ(script_args_list=[]):
+    parser = argparse.ArgumentParser()
+    for args, kwargs in script_args_list:
+        parser.add_argument(*args, **kwargs)
+    args = parser.parse_args()
+    print(f"Command Line Arguments: {args}")
+    os.environ.update(vars(args))

deltacat/examples/hello_world.py

@@ -0,0 +1,27 @@
+import ray
+import deltacat
+import daft
+import pyiceberg
+
+
+def print_package_version_info():
+    print(f"DeltaCAT Version: {deltacat.__version__}")
+    print(f"PyIceberg Version: {pyiceberg.__version__}")
+    print(f"Ray Version: {ray.__version__}")
+    print(f"Daft Version: {daft.__version__}")
+
+
+@ray.remote
+def hello_worker():
+    print("Hello, Worker!")
+    print_package_version_info()
+
+
+def run():
+    print("Hello, Driver!")
+    print_package_version_info()
+    hello_worker.remote()
+
+
+if __name__ == "__main__":
+    run()