deltacat 2.0.0b7__py3-none-any.whl → 2.0.0b10__py3-none-any.whl

deltacat/compute/converter/utils/convert_task_options.py

@@ -4,7 +4,10 @@ from deltacat.exceptions import RetryableError
 AVERAGE_FILE_PATH_COLUMN_SIZE_BYTES = 80
 AVERAGE_POS_COLUMN_SIZE_BYTES = 4
 XXHASH_BYTE_PER_RECORD = 8
-MEMORY_BUFFER_RATE = 1.2
+MEMORY_BUFFER_RATE = 2
+# TODO: Add audit info to check this number in practice
+# Worst case 2 as no duplicates exists across all pk
+PYARROW_AGGREGATE_MEMORY_MULTIPLIER = 2
 
 
 def estimate_fixed_hash_columns(hash_value_size_bytes_per_record, total_record_count):
@@ -13,8 +16,8 @@ def estimate_fixed_hash_columns(hash_value_size_bytes_per_record, total_record_c
 
 def get_total_record_from_iceberg_files(iceberg_files_list):
     total_record_count = 0
-    for iceberg_files in iceberg_files_list:
-        total_record_count += sum(file.record_count for file in iceberg_files)
+    # file are in form of tuple (sequence_number, DataFile)
+    total_record_count += sum(file[1].record_count for file in iceberg_files_list)
     return total_record_count
 
 
@@ -68,7 +71,8 @@ def _get_task_options(
         task_opts["resources"] = ray_custom_resources
 
     task_opts["max_retries"] = 3
-
+    task_opts["num_cpus"] = 1
+    task_opts["resources"] = {"convert_task": 1}
     # List of possible botocore exceptions are available at
     # https://github.com/boto/botocore/blob/develop/botocore/exceptions.py
     task_opts["retry_exceptions"] = [RetryableError]
@@ -76,13 +80,38 @@ def _get_task_options(
     return task_opts
 
 
-def convert_resource_options_provider(index, files_for_each_bucket):
-    (
-        data_files_list,
-        equality_delete_files_list,
-        position_delete_files_list,
-    ) = files_for_each_bucket[1]
-    memory_requirement = estimate_convert_remote_option_resources(
-        data_files_list, equality_delete_files_list
+def estimate_dedupe_memory(all_data_files_for_dedupe):
+    dedupe_record_count = get_total_record_from_iceberg_files(all_data_files_for_dedupe)
+    produced_pos_memory_required = estimate_iceberg_pos_delete_additional_columns(
+        ["file_path", "pos"], dedupe_record_count
+    )
+    download_pk_memory_required = estimate_fixed_hash_columns(
+        XXHASH_BYTE_PER_RECORD, dedupe_record_count
     )
-    return _get_task_options(memory=memory_requirement)
+    memory_required_by_dedupe = (
+        produced_pos_memory_required + download_pk_memory_required
+    ) * PYARROW_AGGREGATE_MEMORY_MULTIPLIER
+    memory_with_buffer = memory_required_by_dedupe * MEMORY_BUFFER_RATE
+    return memory_with_buffer
+
+
+def convert_resource_options_provider(index, convert_input_files):
+    applicable_data_files = convert_input_files.applicable_data_files
+    applicable_equality_delete_files = (
+        convert_input_files.applicable_equality_delete_files
+    )
+    all_data_files_for_dedupe = convert_input_files.all_data_files_for_dedupe
+    total_memory_required = 0
+    if applicable_data_files and applicable_equality_delete_files:
+        memory_requirement_for_convert_equality_deletes = (
+            estimate_convert_remote_option_resources(
+                applicable_data_files, applicable_equality_delete_files
+            )
+        )
+        total_memory_required += memory_requirement_for_convert_equality_deletes
+    if all_data_files_for_dedupe:
+        memory_requirement_for_dedupe = estimate_dedupe_memory(
+            all_data_files_for_dedupe
+        )
+        total_memory_required += memory_requirement_for_dedupe
+    return _get_task_options(memory=total_memory_required)

deltacat/compute/converter/utils/iceberg_columns.py

@@ -80,3 +80,8 @@ def append_global_record_idx_column(
         pa.array(ordered_record_indices, _GLOBAL_RECORD_IDX_COLUMN_TYPE),
     )
     return table
+
+
+_IDENTIFIER_COLUMNS_HASH_COLUMN_NAME = _get_iceberg_col_name(
+    "identifier_columns_hashed"
+)

deltacat/compute/converter/utils/io.py

@@ -1,26 +1,43 @@
+import logging
+from deltacat import logs
 import deltacat.compute.converter.utils.iceberg_columns as sc
 import daft
+from deltacat.utils.daft import _get_s3_io_config
+from daft import TimeUnit
+import pyarrow as pa
+from deltacat.utils.pyarrow import sliced_string_cast
+from deltacat.compute.converter.constants import IDENTIFIER_FIELD_DELIMITER
+
+import pyarrow.compute as pc
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 def download_data_table_and_append_iceberg_columns(
-    file, columns_to_download, additional_columns_to_append, sequence_number
+    file,
+    columns_to_download,
+    additional_columns_to_append,
+    sequence_number,
+    s3_client_kwargs,
 ):
-    # TODO; add S3 client kwargs
     table = download_parquet_with_daft_hash_applied(
-        identify_columns=columns_to_download, file=file, s3_client_kwargs={}
+        identifier_columns=columns_to_download,
+        file=file,
+        s3_client_kwargs=s3_client_kwargs,
     )
+
     if sc._FILE_PATH_COLUMN_NAME in additional_columns_to_append:
         table = sc.append_file_path_column(table, file.file_path)
     if sc._ORDERED_RECORD_IDX_COLUMN_NAME in additional_columns_to_append:
         record_idx_iterator = iter(range(len(table)))
         table = sc.append_record_idx_col(table, record_idx_iterator)
+
     return table
 
 
 def download_parquet_with_daft_hash_applied(
-    identify_columns, file, s3_client_kwargs, **kwargs
+    identifier_columns, file, s3_client_kwargs, **kwargs
 ):
-    from daft import TimeUnit
 
     # TODO: Add correct read kwargs as in:
     #  https://github.com/ray-project/deltacat/blob/383855a4044e4dfe03cf36d7738359d512a517b4/deltacat/utils/daft.py#L97
@@ -29,15 +46,69 @@ def download_parquet_with_daft_hash_applied(
         kwargs.get("coerce_int96_timestamp_unit", "ms")
     )
 
-    from deltacat.utils.daft import _get_s3_io_config
-
     # TODO: Use Daft SHA1 hash instead to minimize probably of data corruption
     io_config = _get_s3_io_config(s3_client_kwargs=s3_client_kwargs)
-    df = daft.read_parquet(
+    df = daft_read_parquet(
         path=file.file_path,
         io_config=io_config,
         coerce_int96_timestamp_unit=coerce_int96_timestamp_unit,
     )
-    df = df.select(daft.col(identify_columns[0]).hash())
-    arrow_table = df.to_arrow()
-    return arrow_table
+
+    hash_column = concatenate_hashed_identifier_columns(
+        df=df, identifier_columns=identifier_columns
+    )
+
+    table = pa.Table.from_arrays(
+        [hash_column], names=[sc._IDENTIFIER_COLUMNS_HASH_COLUMN_NAME]
+    )
+
+    return table
+
+
+def daft_read_parquet(path, io_config, coerce_int96_timestamp_unit):
+    df = daft.read_parquet(
+        path=path,
+        io_config=io_config,
+        coerce_int96_timestamp_unit=coerce_int96_timestamp_unit,
+    )
+    return df
+
+
+def concatenate_hashed_identifier_columns(df, identifier_columns):
+    pk_hash_columns = []
+    previous_hash_column_length = None
+    for i in range(len(identifier_columns)):
+        pk_hash_column = df.select(daft.col(identifier_columns[i]).hash())
+        pk_hash_column_arrow = pk_hash_column.to_arrow()
+
+        # Assert that each hash column downloaded are same length to ensure we don't create mismatch between columns.
+        if not previous_hash_column_length:
+            previous_hash_column_length = len(pk_hash_column_arrow)
+        else:
+            assert previous_hash_column_length == len(pk_hash_column_arrow), (
+                f"Identifier column Length mismatch: {identifier_columns[i]} has length {len(pk_hash_column_arrow)} "
+                f"but expected {previous_hash_column_length}."
+            )
+            previous_hash_column_length = len(pk_hash_column_arrow)
+
+        # Convert identifier from different datatypes to string here
+        pk_hash_column_str = sliced_string_cast(
+            pk_hash_column_arrow[identifier_columns[i]]
+        )
+        assert len(pk_hash_column_str) == previous_hash_column_length, (
+            f"Casting column Length mismatch: {identifier_columns[i]} has length {len(pk_hash_column_str)} after casting, "
+            f"before casting length: {previous_hash_column_length}."
+        )
+
+        pk_hash_columns.append(pk_hash_column_str)
+
+    pk_hash_columns.append(IDENTIFIER_FIELD_DELIMITER)
+    pk_hash_columns_concatenated = pc.binary_join_element_wise(
+        *pk_hash_columns, null_handling="replace"
+    )
+    assert len(pk_hash_columns_concatenated) == previous_hash_column_length, (
+        f"Concatenated column Length mismatch: Final concatenated identifier column has length {len(pk_hash_columns_concatenated)}, "
+        f"before concatenating length: {previous_hash_column_length}."
+    )
+
+    return pk_hash_columns_concatenated

deltacat/compute/converter/utils/s3u.py

@@ -11,7 +11,7 @@ from deltacat.types.tables import (
     get_table_length,
     TABLE_CLASS_TO_SLICER_FUNC,
 )
-from typing import Optional, Dict, Any, List
+from typing import Optional, Dict, Any
 from deltacat.exceptions import RetryableError
 from deltacat.storage import (
     DistributedDataset,
@@ -59,7 +59,7 @@ def upload_table_with_retry(
     max_records_per_file: Optional[int] = 4000000,
     s3_file_system=None,
     **s3_client_kwargs,
-) -> List[str]:
+):
     """
     Writes the given table to 1 or more S3 files and return Redshift
     manifest entries describing the uploaded files.
@@ -77,7 +77,7 @@ def upload_table_with_retry(
         s3_file_system = get_s3_file_system(content_type=content_type)
     capture_object = CapturedBlockWritePaths()
     block_write_path_provider = UuidBlockWritePathProvider(
-        capture_object=capture_object
+        capture_object=capture_object, base_path=s3_url_prefix
     )
     s3_table_writer_func = get_table_writer(table)
     table_record_count = get_table_length(table)
@@ -110,7 +110,16 @@ def upload_table_with_retry(
             )
     del block_write_path_provider
     write_paths = capture_object.write_paths()
-    return write_paths
+    s3_write_paths = []
+    for path in write_paths:
+        s3_write_path = construct_s3_url(path)
+        s3_write_paths.append(s3_write_path)
+    return s3_write_paths
+
+
+def construct_s3_url(path):
+    if path:
+        return f"s3://{path}"
 
 
 def upload_table(

deltacat/compute/jobs/client.py

@@ -0,0 +1,404 @@
+# from deltacat.compute import index
+import subprocess
+import socket
+import os
+import time
+import re
+
+import deltacat as dc
+
+from dataclasses import dataclass
+
+from typing import Set, Optional, Dict, Any, Union
+
+from ray.job_submission import JobSubmissionClient, JobStatus
+
+from deltacat.utils.performance import timed_invocation
+
+
+def _run_cmd(cmd: str) -> None:
+    exit_code = int(os.system(cmd))
+    assert exit_code == 0, f"`{cmd}` failed. Exit code: {exit_code}"
+
+
+def _ray_up(cluster_cfg: str, restart_only: bool = False) -> None:
+    restart_flag = "--no-restart" if not restart_only else "--restart-only"
+    print(f"Starting Ray cluster from '{cluster_cfg}'")
+    _run_cmd(
+        f"ray up '{cluster_cfg}' -y --no-config-cache {restart_flag} --disable-usage-stats"
+    )
+    print(f"Started Ray cluster from '{cluster_cfg}'")
+
+
+def _is_port_in_use(port: Union[int, str]) -> bool:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        return s.connect_ex(("localhost", int(port))) == 0
+
+
+def _is_dashboard_running(port: Union[int, str]) -> bool:
+    return _is_port_in_use(port)
+
+
+def _ray_dashboard_up(
+    cluster_cfg: str, port: Union[str, int], timeout_seconds=15
+) -> None:
+    print(f"Starting Ray Dashboard for Ray cluster '{cluster_cfg}'")
+    _run_cmd(f"ray dashboard '{cluster_cfg}' --port {port} &")
+    start = time.monotonic()
+    dashboard_is_up = False
+    while time.monotonic() - start <= timeout_seconds:
+        if _is_dashboard_running(port):
+            dashboard_is_up = True
+            break
+        time.sleep(0.1)
+    if not dashboard_is_up:
+        raise TimeoutError(
+            f"Timed out after waiting {timeout_seconds} seconds for dashboard "
+            f"to establish connection on port {port}."
+        )
+    print(f"Started Ray Dashboard for Ray cluster '{cluster_cfg}'")
+
+
+def _get_head_node_ip(cluster_cfg: str) -> str:
+    print(f"Getting Ray cluster head node IP for '{cluster_cfg}'")
+    cmd = f"ray get-head-ip '{cluster_cfg}'"
+    proc = subprocess.run(
+        cmd,
+        shell=True,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    # the head node IP should be the last line printed to stdout
+    head_node_ip = proc.stdout.splitlines()[-1]
+    if not re.match(
+        r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$",
+        head_node_ip,
+    ):
+        print(
+            f"Failed to find Ray Head Node IP Address in `{cmd}` "
+            f"output: {proc.stdout}"
+        )
+        raise RuntimeError("No Ray Head Node IP Address Found")
+    print(f"Ray cluster head node IP for '{cluster_cfg}': {head_node_ip}")
+    return head_node_ip
+
+
+def _ray_down_cmd(cluster_cfg: str) -> str:
+    return f"ray down '{cluster_cfg}' -y"
+
+
+def _ray_down(cluster_cfg: str) -> None:
+    print(f"Destroying Ray cluster for '{cluster_cfg}'")
+    _run_cmd(_ray_down_cmd(cluster_cfg))
+    print(f"Destroyed Ray cluster for '{cluster_cfg}'")
+
+
+def _ray_cluster_running(cluster_cfg: str) -> bool:
+    try:
+        _get_head_node_ip(cluster_cfg)
+    except Exception as e:
+        print(f"Get Head Node IP Failed with Exception: {e}")
+        print(f"Assuming Ray Cluster is Not Running")
+        return False
+    return True
+
+
+@dataclass(frozen=True)
+class DeltaCatJobRunResult:
+    job_id: str
+    job_status: JobStatus
+    job_logs: Any
+
+
+class DeltaCatJobClient(JobSubmissionClient):
+    @staticmethod
+    def of(
+        cluster_cfg_file_path: str = "./deltacat.yaml",
+        *,
+        launch_cluster: bool = True,
+        start_dashboard: bool = True,
+        restart_ray: bool = False,
+        head_node_ip: str = None,
+        dashboard_wait_time_seconds: int = 30,
+        port: Union[int, str] = "8265",
+    ):
+        job_submission_client_url = None
+        try:
+            # launch Ray cluster if necessary
+            if cluster_cfg_file_path:
+                if launch_cluster:
+                    if not _ray_cluster_running(cluster_cfg_file_path) or restart_ray:
+                        _ray_up(cluster_cfg_file_path)
+                elif restart_ray:
+                    if _ray_cluster_running(cluster_cfg_file_path):
+                        _ray_up(cluster_cfg_file_path, restart_ray)
+                    else:
+                        raise RuntimeError(
+                            f"Cannot Restart Ray: Ray Cluster for "
+                            f"`{cluster_cfg_file_path}` not found."
+                        )
+                dashboard_running = _is_dashboard_running(port)
+                if not dashboard_running and start_dashboard:
+                    _ray_dashboard_up(
+                        cluster_cfg=cluster_cfg_file_path,
+                        port=port,
+                        timeout_seconds=dashboard_wait_time_seconds,
+                    )
+                    dashboard_running = True
+                if not head_node_ip:
+                    head_node_ip = (
+                        "127.0.0.1"
+                        # use dashboard port forwarding on localhost
+                        if dashboard_running
+                        # fetch the remote head node IP
+                        else _get_head_node_ip(cluster_cfg_file_path)
+                    )
+            else:
+                head_node_ip = "127.0.0.1"
+            job_submission_client_url = f"http://{head_node_ip}:{port}"
+            print(
+                f"Initializing Ray Job Submission Client with URL: "
+                f"{job_submission_client_url}"
+            )
+            client = JobSubmissionClient(f"http://{head_node_ip}:{port}")
+            # the below class change is safe as long as we only add new methods
+            # to the wrapped JobSubmissionClient that don't alter its internal
+            # state
+            client.__class__ = DeltaCatJobClient
+            return client
+        except Exception as e:
+            print(f"Unexpected error while initializing Ray Job Client: {e}")
+            if job_submission_client_url:
+                print(
+                    f"Please ensure that Ray was installed with a job server "
+                    f'enabled via `pip install -U "ray[default]"` and '
+                    f"that http://{head_node_ip}:{port} is accessible. You "
+                    f"can optionally run `ray dashboard` to forward the "
+                    f"remote Ray head node port to a local port (default 8265) "
+                    f'then run `ray_job_client("127.0.0.1", 8265)` '
+                    f"to connect via localhost."
+                )
+            if cluster_cfg_file_path:
+                print(
+                    f"If you're done submitting jobs, ensure that the remote "
+                    f"Ray Cluster is shut down by running: "
+                    f"{_ray_down_cmd(cluster_cfg_file_path)}"
+                )
+            raise e
+
+    def run_job(
+        self,
+        *,
+        entrypoint: str,
+        runtime_env: Optional[Dict[str, Any]] = None,
+        timeout_seconds: int = 600,
+        **kwargs,
+    ) -> DeltaCatJobRunResult:
+        """
+        Synchronously submit and run a Ray job. This method combines Ray job submission and monitoring by submitting
+        the job to the Ray Job Server, waiting for the job to complete,
+        validating the job's terminal status, retrieving and returning job run
+        result information if successful.
+
+        Args:
+            entrypoint: The entry point for the job to be executed (module
+                or script to run)
+            runtime_env: Runtime environment configuration for the job.
+                Some commonly used keys include `working_dir` (directory
+                containing the job code), `pip` (list of pip packages to
+                install), and `env_vars` (environment variables for the job).
+            timeout_seconds: Maximum time in seconds to wait for job completion.
+                Default to 600 seconds (10 minutes).
+            kwargs: Additional keyword arguments to pass to the job submission.
+
+        Returns:
+            Final results from the successful job run execution.
+
+        Raises:
+            RuntimeError: If the job fails or terminates with status other
+                than SUCCEEDED.
+            TimeoutError: If the job doesn't complete within the specified
+                timeout period
+
+        Example:
+            >>> client = job_client()
+            >>> logs = client.run_job(
+            ...     # Shell command to run job
+            ...     entrypoint="my_script.py",
+            ...     runtime_env={
+            ...         # Path to the local directory containing my_script.py
+            ...         "working_dir": "./",
+            ...         # Pip dependencies to install
+            ...         "pip": ["pandas", "numpy"],
+            ...         # System environment variables to set
+            ...         "env_vars": {"DATA_PATH": "/path/to/data"},
+            ...     },
+            ...     timeout_seconds=1200
+            ... )
+        """
+
+        job_id = self.submit_job(
+            entrypoint=entrypoint,
+            runtime_env=runtime_env,
+            **kwargs,
+        )
+        job_status, latency = timed_invocation(
+            self.await_job,
+            job_id,
+            timeout_seconds=timeout_seconds,
+        )
+        job_logs = self.get_job_logs(job_id)
+        if job_status != JobStatus.SUCCEEDED:
+            print(f"Job `{job_id}` logs: ")
+            print(job_logs)
+            raise RuntimeError(f"Job `{job_id}` terminated with status: {job_status}")
+        return DeltaCatJobRunResult(
+            job_id=job_id,
+            job_status=job_status,
+            job_logs=job_logs,
+        )
+
+    def await_job(
+        self,
+        job_id: str,
+        await_status: Set[JobStatus] = {
+            JobStatus.SUCCEEDED,
+            JobStatus.STOPPED,
+            JobStatus.FAILED,
+        },
+        *,
+        timeout_seconds: int = 600,
+    ) -> JobStatus:
+        """
+        Polls a job's status until it matches the desired status or times out.
+
+        This function continuously checks the status of a specified job using the
+        provided client. It will keep polling until either the desired status is
+        reached or the timeout period expires.
+
+        Args:
+            job_id: The unique identifier of the job to monitor.
+            await_status: Set of :class:`ray.job_submission.JobStatus` to wait for.
+                The function will return when the job reaches any of these states.
+            timeout_seconds: Maximum time to wait in seconds.
+                Defaults to 600 seconds (10 minutes).
+
+        Returns:
+            The final status of the job.
+
+        Raises:
+            TimeoutError: If the desired status is not reached within the
+            specified timeout period.
+
+        Example:
+            >>>
+            >>> client = job_client()
+            >>> job_id = client.submit_job(
+            >>>     # Shell command to run job
+            >>>     entrypoint=f"python copy.py --source '{source}' --dest '{dest}'",
+            >>>     # Path to the local directory containing copy.py
+            >>>     runtime_env={"working_dir": "./"},
+            >>> )
+            >>> # wait for the job to reach a terminal state
+            >>> client.await_job(job_id)
+        """
+        start = time.monotonic()
+        terminal_status = None
+        while time.monotonic() - start <= timeout_seconds:
+            status = self.get_job_status(job_id)
+            if status in await_status:
+                terminal_status = status
+                break
+            time.sleep(0.1)
+        if not terminal_status:
+            self.stop_job(job_id)
+            raise TimeoutError(
+                f"Timed out after waiting {timeout_seconds} seconds for job "
+                f"`{job_id}` status: {status}"
+            )
+        return terminal_status
+
+
+def local_job_client(*args, **kwargs) -> DeltaCatJobClient:
+    """
+    Create a Ray Job Client that can be used to submit jobs to a local Ray
+    cluster. Initializes Ray if it's not already running.
+
+    Args:
+        *args: Positional arguments to pass to `ray.init()`.
+        **kwargs: Keyword arguments to pass to `ray.init()`.
+    Returns:
+        DeltaCatJobClient: A client instance that can be used to submit and
+            manage local Ray jobs.
+
+    Raises:
+        RuntimeError: If a local Ray Job Server cannot be found.
+    """
+    if not dc.is_initialized():
+        context = dc.init(*args, **kwargs)
+    else:
+        context = dc.init(ray_init_args={"ignore_reinit_error": True})
+    if context.dashboard_url:
+        head_node_ip, port = context.dashboard_url.split(":")
+    else:
+        # the Ray Dashboard URL is also the Ray Job Server URL
+        raise RuntimeError(
+            "Ray Job Server not found! Please reinstall Ray using "
+            "`pip install -U `ray[default]`"
+        )
+    return DeltaCatJobClient.of(
+        None,
+        launch_cluster=False,
+        start_dashboard=False,
+        head_node_ip=head_node_ip,
+        port=port,
+    )
+
+
+def job_client(
+    cluster_cfg_file_path: str = "./deltacat.yaml",
+    *,
+    launch_cluster: bool = True,
+    start_dashboard: bool = True,
+    restart_ray: bool = False,
+    head_node_ip: str = None,
+    dashboard_wait_time_seconds: int = 15,
+    port: Union[str, int] = "8265",
+) -> DeltaCatJobClient:
+    """
+    Create a DeltaCAT Job Client that can be used to submit jobs to a remote Ray cluster.
+
+    Args:
+        cluster_cfg_file_path: Path to the Ray Cluster Launcher
+            Config file. Defaults to "./deltacat.yaml".
+        launch_cluster : Whether to launch a new Ray cluster.
+            Defaults to True.
+        start_dashboard: Whether to start the Ray dashboard.
+            Defaults to True.
+        restart_ray: Whether to restart Ray if it's already
+            running. Defaults to False.
+        head_node_ip: IP address of the Ray cluster head node.
+            If None, will use the configuration from the cluster config file.
+            Defaults to None.
+        dashboard_wait_time_seconds: Time in seconds to wait for the Ray
+            dashboard to start if `start_dashboard` is True.
+        port: Port number for the Ray
+            dashboard/job server. Defaults to "8265".
+
+    Returns:
+        DeltaCatJobClient: A client instance that can be used to submit and
+            manage jobs on the Ray cluster.
+
+    Raises:
+        RuntimeError: If the Ray Job Server is not found.
+    """
+    return DeltaCatJobClient.of(
+        cluster_cfg_file_path,
+        launch_cluster=launch_cluster,
+        start_dashboard=start_dashboard,
+        restart_ray=restart_ray,
+        head_node_ip=head_node_ip,
+        dashboard_wait_time_seconds=dashboard_wait_time_seconds,
+        port=port,
+    )

deltacat/constants.py

@@ -44,10 +44,9 @@ DELTACAT_ROOT = env_string(
 )
 
 # CLI Args
-METAFILE_FORMAT_KEY = "METAFILE_FORMAT"
 METAFILE_FORMAT_JSON = "json"
 METAFILE_FORMAT_MSGPACK = "msgpack"
-METAFILE_FORMAT = env_string(METAFILE_FORMAT_KEY, METAFILE_FORMAT_MSGPACK)
+METAFILE_FORMAT = env_string("METAFILE_FORMAT", METAFILE_FORMAT_MSGPACK)
 SUPPORTED_METAFILE_FORMATS = [METAFILE_FORMAT_JSON, METAFILE_FORMAT_MSGPACK]
 METAFILE_EXT = {
     "json": ".json",
@@ -95,11 +94,12 @@ RUNNING_TXN_DIR_NAME: str = "running"
 FAILED_TXN_DIR_NAME: str = "failed"
 SUCCESS_TXN_DIR_NAME: str = "success"
 TXN_PART_SEPARATOR = "_"
+
 # Storage interface defaults
 # These defaults should be applied in catalog interface implementations
 # Storage interface implementations should be agnostic to defaults and require full information
-DEFAULT_CATALOG = "DEFAULT"
-DEFAULT_NAMESPACE = "DEFAULT"
+DEFAULT_CATALOG = "default"
+DEFAULT_NAMESPACE = "default"
 DEFAULT_TABLE_VERSION = "1"
 DEFAULT_STREAM_ID = "stream"
 DEFAULT_PARTITION_ID = "partition"