data-prep-toolkit 0.2.2.dev0__py3-none-any.whl → 0.2.2.dev1__py3-none-any.whl

data_prep_toolkit-0.2.2.dev1.dist-info/METADATA

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: data_prep_toolkit
+Version: 0.2.2.dev1
+Summary: Data Preparation Toolkit Library for Ray and Python
+Author-email: Maroun Touma <touma@us.ibm.com>
+License: Apache-2.0
+Keywords: data,data preprocessing,data preparation,llm,generative,ai,fine-tuning,llmapps
+Requires-Python: <3.13,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: numpy<1.29.0
+Requires-Dist: pyarrow==16.1.0
+Requires-Dist: boto3==1.34.69
+Requires-Dist: argparse
+Requires-Dist: mmh3
+Requires-Dist: psutil
+Provides-Extra: dev
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: pytest>=7.3.2; extra == "dev"
+Requires-Dist: pytest-dotenv>=0.5.2; extra == "dev"
+Requires-Dist: pytest-env>=1.0.0; extra == "dev"
+Requires-Dist: pre-commit>=3.3.2; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.10.0; extra == "dev"
+Requires-Dist: moto==5.0.5; extra == "dev"
+Requires-Dist: markupsafe==2.0.1; extra == "dev"
+Provides-Extra: ray
+Requires-Dist: ray[default]==2.36.1; extra == "ray"
+Requires-Dist: fastapi>=0.110.2; extra == "ray"
+Requires-Dist: pillow>=10.3.0; extra == "ray"
+Provides-Extra: spark
+Requires-Dist: pyspark>=3.5.2; extra == "spark"
+Requires-Dist: psutil>=6.0.0; extra == "spark"
+

{data_prep_toolkit-0.2.2.dev0.dist-info → data_prep_toolkit-0.2.2.dev1.dist-info}/RECORD

@@ -18,7 +18,7 @@ data_processing/runtime/pure_python/runtime_configuration.py,sha256=a4vSY98HfRm2
 data_processing/runtime/pure_python/transform_file_processor.py,sha256=PYWNUSeb6i6q6Ov7nE0jXQfHIhp1u9adArEU3mQ7B24,4394
 data_processing/runtime/pure_python/transform_invoker.py,sha256=lAG7tfyJyNqtwRB15-db4HJOQsBhT6JahLmjUFQFCRk,5192
 data_processing/runtime/pure_python/transform_launcher.py,sha256=BDctJnYlR9OVzGCzMwg2cEuGdnV3E9fvhUgoyslvK8k,2447
-data_processing/runtime/pure_python/transform_orchestrator.py,sha256=1ChVBTdyM6SwWVyx1HuSqWXRU8UywAQWMiQWaF43sD8,8834
+data_processing/runtime/pure_python/transform_orchestrator.py,sha256=OIQzOL0jT-3ahT7aDs6suySkoEhmvM_T4C_qMDt0JSQ,9468
 data_processing/runtime/pure_python/transform_runtime.py,sha256=pWvuGJGAB6M798LJU3FZBG6l35VQCuhsh-SyzSf9ok0,2558
 data_processing/test_support/__init__.py,sha256=O4lySih15vkOYUSa3uhTaoYw0RrV4rM_sUd691JEuVU,83
 data_processing/test_support/abstract_test.py,sha256=gZ51wnWITEAyb8BzA2WFCM0quJBxQrlD7WBwUfIsWEA,12875
@@ -45,7 +45,28 @@ data_processing/utils/transform_configuration.json,sha256=6YBw0Hk2mokY6JBn1kR6L9
 data_processing/utils/transform_configurator.py,sha256=9OHSCQ8rFSoDdMW6ZCHYdNe6thRwV9zOaRPnLkWNMYE,3601
 data_processing/utils/transform_utils.py,sha256=KGNioN35B1i1h-MIsfm3QvXLlU1aGXimheva7NbUhMM,8496
 data_processing/utils/unrecoverable.py,sha256=svNdVzQaArnf8GdLvB2nP9miv7kYe3bDfFRW--SWvbU,171
-data_prep_toolkit-0.2.2.dev0.dist-info/METADATA,sha256=6WIKrryg4jGjhscucT4P-Jhr8mxP0L8sQz6nywPPaRk,1900
-data_prep_toolkit-0.2.2.dev0.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-data_prep_toolkit-0.2.2.dev0.dist-info/top_level.txt,sha256=OZPYA4BEseUcpIjvbiPpE3Hdcz5cVWwO1Kji5oed2A4,16
-data_prep_toolkit-0.2.2.dev0.dist-info/RECORD,,
+data_processing_ray/runtime/ray/__init__.py,sha256=vjQOvb_OJNq3c1F_tG3WjO-pciY77Z1lETO2Ha_GVbw,784
+data_processing_ray/runtime/ray/execution_configuration.py,sha256=C9YFixlATr7PPpkVQ0WzjCCPTWFuP80W2rnzY1bbp5I,4628
+data_processing_ray/runtime/ray/ray_utils.py,sha256=eDPm-pybPOELjKkvoz3l-qFU-k1Iwh-giGlXULiZjEk,10212
+data_processing_ray/runtime/ray/runtime_configuration.py,sha256=js9dXwdxjYbSigMC49F07XmbLjmj9HiipPE6BDaIGfA,1691
+data_processing_ray/runtime/ray/transform_file_processor.py,sha256=eR814VvfmPOlvyv_FU7eyt0HRqIzkkvACURxJCK-xrM,2335
+data_processing_ray/runtime/ray/transform_invoker.py,sha256=apfH8uilpm9sJ4IpHgiNdIzcH_IHGQba5fui4cCfolk,4026
+data_processing_ray/runtime/ray/transform_launcher.py,sha256=oxI3MFZI_-LzTwHbrHBIUqJ0htnliKBuALt86qijRwU,4304
+data_processing_ray/runtime/ray/transform_orchestrator.py,sha256=FZl7NM0eU1SxOcavZm4lru3laCswACPB1rjk3KK3FtY,7102
+data_processing_ray/runtime/ray/transform_runtime.py,sha256=0-b5syOW9zNnZxmMHDdwPo_pvoqDBiM5dHCgSakZhGQ,2531
+data_processing_ray/runtime/ray/transform_statistics.py,sha256=cxrSQVnzRBCGS68IoiVGLoRBWBxPBSFFMDiT29FNt0g,3749
+data_processing_ray/test_support/transform/__init__.py,sha256=CKk-J3aEwH7OgDardyUEbLjlWaZWLUBs93PdukT4Rbc,100
+data_processing_ray/test_support/transform/noop_transform.py,sha256=ZTx09M9vNOaqrVzeuT2VmWM-IF4Upip0g0EtbHaOn-0,1588
+data_processing_spark/runtime/spark/__init__.py,sha256=bhY1xI9lL0GR2v1APahlhC5sh5rdVcGhQbWN4yoXApw,1233
+data_processing_spark/runtime/spark/execution_configuration.py,sha256=BqxUlpXFdHRK-csO2jaJJtktyKbcMtjIn3sjAPBfO58,3643
+data_processing_spark/runtime/spark/runtime_configuration.py,sha256=uABzBvzzFZ5HA_lGYEBFRd1qViMj5sbpKtCSLy64riM,1705
+data_processing_spark/runtime/spark/transform_file_processor.py,sha256=sdDBZZyqCqyKaJmEqZh1QzIqCDkLQzqV0dcAI5TRBjo,2611
+data_processing_spark/runtime/spark/transform_launcher.py,sha256=1PZ-N4Wy2Qqiqr2z9S1xV88cNsAoHrmmuPadiOakJLM,2479
+data_processing_spark/runtime/spark/transform_orchestrator.py,sha256=HadnLNx_icy2n7CXOwqLiUA7vjV-gOvajxE0AQU3_NM,6645
+data_processing_spark/runtime/spark/transform_runtime.py,sha256=IKChGY1uGxFlAqZaL-XeSv_J3BMm3nev9MAs0NTT8og,2506
+data_processing_spark/test_support/transform/__init__.py,sha256=v58HbP2x9KF8MG8SOGWjodrTjU57KXlL0aPPB7z8KQQ,755
+data_processing_spark/test_support/transform/noop_transform.py,sha256=0FR3o-LnXf-UFS5gU0j-i4LVlw1mHDxGaPI40dkkIKY,1694
+data_prep_toolkit-0.2.2.dev1.dist-info/METADATA,sha256=QiZEK2qc8or6csEZk_weYltxiDRFvDb0chVpwLMCMrU,1235
+data_prep_toolkit-0.2.2.dev1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+data_prep_toolkit-0.2.2.dev1.dist-info/top_level.txt,sha256=XGMDmY55_pe5KeRWvO0un9a640e2v99tzbBBtjNybPM,58
+data_prep_toolkit-0.2.2.dev1.dist-info/RECORD,,

data_prep_toolkit-0.2.2.dev1.dist-info/top_level.txt

@@ -0,0 +1,3 @@
+data_processing
+data_processing_ray
+data_processing_spark

data_processing/runtime/pure_python/transform_orchestrator.py

@@ -9,9 +9,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ################################################################################
-
+import os
 import time
 import traceback
+import psutil
 from datetime import datetime
 from multiprocessing import Pool
 from typing import Any
@@ -24,12 +25,31 @@ from data_processing.runtime.pure_python import (
     PythonTransformRuntimeConfiguration,
 )
 from data_processing.transform import AbstractBinaryTransform, TransformStatistics
-from data_processing.utils import get_logger
+from data_processing.utils import GB, get_logger
 
 
 logger = get_logger(__name__)
 
 
+@staticmethod
+def _execution_resources() -> dict[str, Any]:
+    """
+    Get Execution resource
+    :return: tuple of cpu/memory usage
+    """
+    # Getting loadover15 minutes
+    load1, load5, load15 = psutil.getloadavg()
+    # Getting memory used
+    mused = round(psutil.virtual_memory()[3] / GB, 2)
+    return {
+        "cpus": round((load15/os.cpu_count()) * 100, 1),
+        "gpus": 0,
+        "memory": mused,
+        "object_store": 0,
+    }
+
+
+
 def orchestrate(
     data_access_factory: DataAccessFactoryBase,
     runtime_config: PythonTransformRuntimeConfiguration,
@@ -43,6 +63,7 @@ def orchestrate(
     :return: 0 - success or 1 - failure
     """
     start_ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    start_time = time.time()
     logger.info(f"orchestrator {runtime_config.get_name()} started at {start_ts}")
     # create statistics
     statistics = TransformStatistics()
@@ -118,6 +139,7 @@ def orchestrate(
             "job_input_params": input_params
             | data_access_factory.get_input_params()
             | execution_config.get_input_params(),
+            "execution_stats": _execution_resources() | {"execution time, min": round((time.time() - start_time) / 60.0, 3)},
             "job_output_stats": stats,
         }
         logger.debug(f"Saving job metadata: {metadata}.")

data_processing_ray/runtime/ray/__init__.py

@@ -0,0 +1,9 @@
+from data_processing_ray.runtime.ray.ray_utils import RayUtils
+from data_processing_ray.runtime.ray.transform_statistics import TransformStatisticsRay
+from data_processing_ray.runtime.ray.transform_runtime import DefaultRayTransformRuntime
+from data_processing_ray.runtime.ray.runtime_configuration import RayTransformRuntimeConfiguration
+from data_processing_ray.runtime.ray.transform_file_processor import RayTransformFileProcessor
+from data_processing_ray.runtime.ray.execution_configuration import RayTransformExecutionConfiguration
+from data_processing_ray.runtime.ray.transform_orchestrator import orchestrate
+from data_processing_ray.runtime.ray.transform_launcher import RayTransformLauncher
+from data_processing_ray.runtime.ray.transform_invoker import execute_ray_transform

data_processing_ray/runtime/ray/execution_configuration.py

@@ -0,0 +1,110 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import argparse
+import ast
+from typing import Any
+
+from data_processing.runtime import TransformExecutionConfiguration
+from data_processing.utils import CLIArgumentProvider, ParamsUtils, get_logger
+
+
+logger = get_logger(__name__)
+
+
+cli_prefix = "runtime_"
+
+
+class RayTransformExecutionConfiguration(TransformExecutionConfiguration):
+    """
+    A class specifying and validating Ray orchestrator configuration
+    """
+
+    def __init__(self, name: str):
+        """
+        Initialization
+        """
+        super().__init__(name=name, print_params=False)
+        self.worker_options = {}
+        self.n_workers = 1
+        self.creation_delay = 0
+
+    def add_input_params(self, parser: argparse.ArgumentParser) -> None:
+        """
+        This method adds transformer specific parameter to parser
+        :param parser: parser
+        :return:
+        """
+        parser.add_argument(f"--{cli_prefix}num_workers", type=int, default=1, help="number of workers")
+
+        help_example_dict = {
+            "num_cpus": ["8", "Required number of CPUs."],
+            "num_gpus": ["1", "Required number of GPUs"],
+            "resources": [
+                '{"special_hardware": 1, "custom_label": 1}',
+                """The complete list can be found at
+           https://docs.ray.io/en/latest/ray-core/api/doc/ray.remote_function.RemoteFunction.options.html#ray.remote_function.RemoteFunction.options
+           and contains accelerator_type, memory, name, num_cpus, num_gpus, object_store_memory, placement_group,
+           placement_group_bundle_index, placement_group_capture_child_tasks, resources, runtime_env,
+           scheduling_strategy, _metadata, concurrency_groups, lifetime, max_concurrency, max_restarts,
+           max_task_retries, max_pending_calls, namespace, get_if_exists""",
+            ],
+        }
+        parser.add_argument(
+            f"--{cli_prefix}worker_options",
+            type=ast.literal_eval,
+            default="{'num_cpus': 0.8}",
+            help="AST string defining worker resource requirements.\n"
+            + ParamsUtils.get_ast_help_text(help_example_dict),
+        )
+        parser.add_argument(f"--{cli_prefix}creation_delay", type=int, default=0, help="delay between actor' creation")
+        return TransformExecutionConfiguration.add_input_params(self, parser=parser)
+
+    def apply_input_params(self, args: argparse.Namespace) -> bool:
+        """
+        Validate transformer specific parameters
+        :param args: user defined arguments
+        :return: True, if validate pass or False otherwise
+        """
+        if not TransformExecutionConfiguration.apply_input_params(self, args=args):
+            return False
+        captured = CLIArgumentProvider.capture_parameters(args, cli_prefix, False)
+        # store parameters locally
+        self.worker_options = captured["worker_options"]
+        self.n_workers = captured["num_workers"]
+        self.creation_delay = captured["creation_delay"]
+        self.job_details = {
+            "job category": "preprocessing",
+            "job name": self.name,
+            "job type": "ray",
+            "job id": captured["job_id"],
+        }
+        # if the user did not define actor max_restarts set it up for fault tolerance
+        if "max_restarts" not in self.worker_options:
+            self.worker_options["max_restarts"] = -1
+
+        # print them
+        logger.info(f"number of workers {self.n_workers} worker options {self.worker_options}")
+        logger.info(f"actor creation delay {self.creation_delay}")
+        logger.info(f"job details {self.job_details}")
+        return True
+
+    def get_input_params(self) -> dict[str, Any]:
+        """
+        get input parameters for job_input_params in metadata
+        :return: dictionary of parameters
+        """
+        return {
+            "number of workers": self.n_workers,
+            "worker options": self.worker_options,
+            "actor creation delay": self.creation_delay,
+        }

data_processing_ray/runtime/ray/ray_utils.py

@@ -0,0 +1,246 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import logging
+import time
+from typing import Any
+
+import ray
+from ray.experimental.state.api import list_actors
+from data_processing.utils import GB, UnrecoverableException
+from ray.actor import ActorHandle
+from ray.exceptions import RayError
+from ray.util.actor_pool import ActorPool
+
+
+class RayUtils:
+    """
+    Class implementing support methods for Ray execution
+    """
+
+    from ray.util.metrics import Gauge
+
+    @staticmethod
+    def get_available_resources(
+        available_cpus_gauge: Gauge = None,
+        available_gpus_gauge: Gauge = None,
+        available_memory_gauge: Gauge = None,
+        object_memory_gauge: Gauge = None,
+    ) -> dict[str, Any]:
+        """
+        Get currently available cluster resources
+        :param available_cpus_gauge: ray Gauge to report available CPU
+        :param available_gpus_gauge: ray Gauge to report available GPU
+        :param available_memory_gauge: ray Gauge to report available memory
+        :param object_memory_gauge: ray Gauge to report available object memory
+        :return: a dict of currently available resources
+        """
+        resources = ray.available_resources()
+        if available_cpus_gauge is not None:
+            available_cpus_gauge.set(int(resources.get("CPU", 0.0)))
+        if available_gpus_gauge is not None:
+            available_gpus_gauge.set(int(resources.get("GPU", 0.0)))
+        if available_memory_gauge is not None:
+            available_memory_gauge.set(resources.get("memory", 0.0) / GB)
+        if object_memory_gauge is not None:
+            object_memory_gauge.set(resources.get("object_store_memory", 0.0) / GB)
+        return {
+            "cpus": int(resources.get("CPU", 0.0)),
+            "gpus": int(resources.get("GPU", 0.0)),
+            "memory": resources.get("memory", 0.0) / GB,
+            "object_store": resources.get("object_store_memory", 0.0) / GB,
+        }
+
+    @staticmethod
+    def get_cluster_resources() -> dict[str, Any]:
+        """
+        Get cluster resources
+        :return: cluster resources
+        """
+        resources = ray.cluster_resources()
+        return {
+            "cpus": int(resources.get("CPU", 0.0)),
+            "gpus": int(resources.get("GPU", 0.0)),
+            "memory": resources.get("memory", 0.0) / GB,
+            "object_store": resources.get("object_store_memory", 0.0) / GB,
+        }
+
+    @staticmethod
+    def get_available_nodes(available_nodes_gauge: Gauge = None) -> int:
+        """
+        Get the list of the alive Ray nodes and optionally expose it to prometheus
+        :param available_nodes_gauge: the gauge used to publish number of available node
+        :return: number of available nodes
+        """
+        # get nodes from Ray
+        nodes = ray.nodes()
+        # filer out available ones
+        nnodes = 0
+        for node in nodes:
+            if node["Alive"]:
+                nnodes += 1
+        return nnodes
+
+    @staticmethod
+    def create_actors(
+        clazz: type, params: dict[str, Any], actor_options: dict[str, Any], n_actors: int, creation_delay: int = 0
+    ) -> list[ActorHandle]:
+        """
+        Create a set of actors
+        :param clazz: actor class, has to be annotated as remote
+        :param params: actor init params
+        :param actor_options: dictionary of actor options.
+        see https://docs.ray.io/en/latest/ray-core/api/doc/ray.actor.ActorClass.options.html
+        :param n_actors: number of actors
+        :param creation_delay - delay between actor's creations
+        :return: a list of actor handles
+        """
+
+        def operator() -> ActorHandle:
+            time.sleep(creation_delay)
+            return clazz.options(**actor_options).remote(params)
+
+        cls_name = clazz.__class__.__name__.replace('ActorClass(', '').replace(')','')
+        actors = [operator() for _ in range(n_actors)]
+        for i in range(120):
+            time.sleep(1)
+            alive = list_actors(filters=[("class_name", "=", cls_name), ("state", "=", "ALIVE")])
+            if len(actors) == len(alive):
+                return actors
+        # failed - raise an exception
+        print(f"created {actors}, alive {alive}")
+        raise UnrecoverableException(f"out of {len(actors)} created actors only {len(alive)} alive")
+
+    @staticmethod
+    def process_files(
+        executors: ActorPool,
+        files: list[str],
+        print_interval: int,
+        files_in_progress_gauge: Gauge,
+        files_completed_gauge: Gauge,
+        available_cpus_gauge: Gauge,
+        available_gpus_gauge: Gauge,
+        available_memory_gauge: Gauge,
+        object_memory_gauge: Gauge,
+        logger: logging.Logger,
+    ) -> int:
+        """
+        Process files
+        :param executors: actor pool of executors
+        :param files: list of files to process
+        :param print_interval: print interval
+        :param files_in_progress_gauge: ray Gauge to report files in process
+        :param files_completed_gauge: ray Gauge to report completed files
+        :param available_cpus_gauge: ray Gauge to report available CPU
+        :param available_gpus_gauge: ray Gauge to report available GPU
+        :param available_memory_gauge: ray Gauge to report available memory
+        :param object_memory_gauge: ray Gauge to report available object memory
+        :param logger: logger
+        :return: number of actors failures
+        """
+        logger.debug("Begin processing files")
+        actor_failures = 0
+        RayUtils.get_available_resources(
+            available_cpus_gauge=available_cpus_gauge,
+            available_gpus_gauge=available_gpus_gauge,
+            available_memory_gauge=available_memory_gauge,
+            object_memory_gauge=object_memory_gauge,
+        )
+        terminate = False
+        running = 0
+        t_start = time.time()
+        completed = 0
+        for path in files:
+            if executors.has_free():  # still have room
+                executors.submit(lambda a, v: a.process_file.remote(v), path)
+                running += 1
+                files_in_progress_gauge.set(running)
+            else:  # need to wait for some actors
+                while True:
+                    # we can have several workers fail here
+                    try:
+                        res = executors.get_next_unordered()
+                        break
+                    except Exception as e:
+                        if isinstance(e, RayError):
+                            # Ray exception - terminate
+                            logger.error(f"Got Ray worker exception {e}, terminating")
+                            terminate = True
+                            break
+                        logger.error(f"Failed to process request worker exception {e}")
+                        actor_failures += 1
+                        completed += 1
+                        break
+                if terminate:
+                    raise UnrecoverableException
+                executors.submit(lambda a, v: a.process_file.remote(v), path)
+
+                completed += 1
+                files_completed_gauge.set(completed)
+                RayUtils.get_available_resources(
+                    available_cpus_gauge=available_cpus_gauge,
+                    available_gpus_gauge=available_gpus_gauge,
+                    available_memory_gauge=available_memory_gauge,
+                    object_memory_gauge=object_memory_gauge,
+                )
+                if completed % print_interval == 0:
+                    logger.info(f"Completed {completed} files in {round((time.time() - t_start)/60., 3)} min")
+        # Wait for completion
+        files_completed_gauge.set(completed)
+        # Wait for completion
+        logger.info(
+            f"Completed {completed} files ({round(100 * completed / len(files), 3)}%)  "
+            f"in {round((time.time() - t_start)/60., 3)} min. Waiting for completion"
+        )
+        while executors.has_next():
+            while True:
+                # we can have several workers fail here
+                try:
+                    executors.get_next_unordered()
+                    break
+                except Exception as e:
+                    logger.error(f"Failed to process request worker exception {e}")
+                    actor_failures += 1
+                    completed += 1
+            running -= 1
+            completed += 1
+            files_in_progress_gauge.set(running)
+            files_completed_gauge.set(completed)
+            RayUtils.get_available_resources(
+                available_cpus_gauge=available_cpus_gauge,
+                available_gpus_gauge=available_gpus_gauge,
+                available_memory_gauge=available_memory_gauge,
+                object_memory_gauge=object_memory_gauge,
+            )
+
+        logger.info(f"Completed processing {completed} files in {round((time.time() - t_start)/60, 3)} min")
+        return actor_failures
+
+    @staticmethod
+    def wait_for_execution_completion(logger: logging.Logger, replies: list[ray.ObjectRef]) -> int:
+        """
+        Wait for all requests completed
+        :param logger: logger to use
+        :param replies: list of request futures
+        :return: None
+        """
+        actor_failures = 0
+        while replies:
+            # Wait for replies
+            try:
+                ready, not_ready = ray.wait(replies)
+            except Exception as e:
+                logger.error(f"Failed to process request worker exception {e}")
+                actor_failures += 1
+                not_ready = replies - 1
+            replies = not_ready
+        return actor_failures

data_processing_ray/runtime/ray/runtime_configuration.py

@@ -0,0 +1,37 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from data_processing.runtime import TransformRuntimeConfiguration
+from data_processing.transform import TransformConfiguration
+from data_processing_ray.runtime.ray import DefaultRayTransformRuntime
+
+
+class RayTransformRuntimeConfiguration(TransformRuntimeConfiguration):
+    def __init__(
+        self,
+        transform_config: TransformConfiguration,
+        runtime_class: type[DefaultRayTransformRuntime] = DefaultRayTransformRuntime,
+    ):
+        """
+        Initialization
+        :param transform_config - base configuration class
+        :param runtime_class: implementation of the transform runtime
+        """
+        super().__init__(transform_config=transform_config)
+        self.runtime_class = runtime_class
+
+    def create_transform_runtime(self) -> DefaultRayTransformRuntime:
+        """
+        Create transform runtime with the parameters captured during apply_input_params()
+        :return: transform runtime object
+        """
+        return self.runtime_class(self.transform_config.get_transform_params())

data_processing_ray/runtime/ray/transform_file_processor.py

@@ -0,0 +1,53 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from typing import Any
+
+import ray
+from data_processing.runtime import AbstractTransformFileProcessor
+from data_processing.utils import UnrecoverableException
+
+
+@ray.remote(scheduling_strategy="SPREAD")
+class RayTransformFileProcessor(AbstractTransformFileProcessor):
+    """
+    This is the class implementing the actual work/actor processing of a single file
+    """
+
+    def __init__(self, params: dict[str, Any]):
+        """
+        Init method
+        :param params: dictionary that has the following key
+            data_access_factory: data access factory
+            transform_class: local transform class
+            transform_params: dictionary of parameters for local transform creation
+            statistics: object reference to statistics
+        """
+        super().__init__(
+            data_access_factory=params.get("data_access_factory", None),
+            transform_parameters=dict(params.get("transform_params", {})),
+        )
+        # Create statistics
+        self.stats = params.get("statistics", None)
+        if self.stats is None:
+            self.logger.error("Transform file processor: statistics is not specified")
+            raise UnrecoverableException("statistics is None")
+        self.transform_params["statistics"] = self.stats
+        # Create local processor
+        try:
+            self.transform = params.get("transform_class", None)(self.transform_params)
+        except Exception as e:
+            self.logger.error(f"Exception creating transform  {e}")
+            raise UnrecoverableException("failed creating transform")
+
+    def _publish_stats(self, stats: dict[str, Any]) -> None:
+        self.stats.add_stats.remote(stats)